envloop-cli 1.4.0__tar.gz → 1.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- envloop_cli-1.7.0/PKG-INFO +195 -0
- envloop_cli-1.7.0/README.pypi.md +182 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/cli.py +35 -12
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/cli_errors.py +5 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/client.py +6 -33
- envloop_cli-1.7.0/envloop/inference.py +35 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/resources.py +0 -4
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/train.py +25 -2
- envloop_cli-1.7.0/envloop/train_runner_cli.py +64 -0
- envloop_cli-1.7.0/envloop/training/__init__.py +4 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/pyproject.toml +2 -5
- envloop_cli-1.4.0/PKG-INFO +0 -122
- envloop_cli-1.4.0/README.pypi.md +0 -109
- envloop_cli-1.4.0/envloop/training/__init__.py +0 -123
- envloop_cli-1.4.0/envloop/training/credentials.py +0 -107
- envloop_cli-1.4.0/envloop/training/sampling_client.py +0 -58
- envloop_cli-1.4.0/envloop/training/service_client.py +0 -165
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/.gitignore +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/__init__.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/agent_runs.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/auth_cli.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/auth_login.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/auth_workspace.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/cli_output.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/config.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/http.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/job_logs.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/lifecycle_cli.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/navigation_resources.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/profile_secrets.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/profile_updates.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/resource_cli.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/rsi_cli.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/sandbox.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/train_log_cli.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/utils/__init__.py +0 -0
- {envloop_cli-1.4.0 → envloop_cli-1.7.0}/envloop/utils/logging.py +0 -0
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: envloop-cli
|
|
3
|
+
Version: 1.7.0
|
|
4
|
+
Summary: EnvLoop CLI and Python SDK for jobs, training, checkpoints and inference
|
|
5
|
+
Project-URL: Homepage, https://envloop.ai
|
|
6
|
+
Project-URL: Documentation, https://docs.envloop.ai
|
|
7
|
+
Project-URL: Repository, https://github.com/EnvLoop/envloop-cli
|
|
8
|
+
Project-URL: Issues, https://github.com/EnvLoop/envloop-cli/issues
|
|
9
|
+
Project-URL: Agent instructions, https://docs.envloop.ai/skill.md
|
|
10
|
+
Requires-Python: >=3.11
|
|
11
|
+
Requires-Dist: tinker==0.34.0
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# envloop-cli: EnvLoop CLI and Python SDK
|
|
15
|
+
|
|
16
|
+
Install the public client to run agent Jobs, inspect Trials, manage training and
|
|
17
|
+
checkpoints, and deploy or call inference endpoints. The `el` and `envloop`
|
|
18
|
+
commands are equivalent. The Python import package is `envloop`.
|
|
19
|
+
|
|
20
|
+
## Version 1.7
|
|
21
|
+
|
|
22
|
+
- Use published Runner resource shortcuts for Train Jobs and fixed Checkpoint inputs for Inference Jobs.
|
|
23
|
+
- Pass `--json` explicitly to `job submit`; failures retain safe error codes and recovery guidance.
|
|
24
|
+
- Submit training through Train Jobs. The retired interactive `envloop.training` interface is removed.
|
|
25
|
+
|
|
26
|
+
After upgrading, check `el --version` and `el job submit --help`. Execution also
|
|
27
|
+
requires a matching platform, project permissions and published Runners; package
|
|
28
|
+
installation alone does not establish those prerequisites. See the
|
|
29
|
+
[PyPI release history](https://pypi.org/project/envloop-cli/#history) for published versions.
|
|
30
|
+
|
|
31
|
+
## Install
|
|
32
|
+
|
|
33
|
+
Use a Python virtual environment:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
python -m pip install --upgrade envloop-cli
|
|
37
|
+
el --help
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The standard installation includes the client dependencies for managed training.
|
|
41
|
+
Check that the SDK can be imported:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
python -c 'from envloop import Client; print("EnvLoop SDK ready")'
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
When you see `EnvLoop SDK ready`, continue to Connect. This check imports the
|
|
48
|
+
client without creating a Job. Submit training through `client.training_jobs`;
|
|
49
|
+
follow the [managed training guide](https://docs.envloop.ai/training/cli-hello-world).
|
|
50
|
+
|
|
51
|
+
Install the client in your Python environment; the platform provides the execution resources.
|
|
52
|
+
|
|
53
|
+
## Connect
|
|
54
|
+
|
|
55
|
+
Use an authorized EnvLoop account and project. Browser authorization:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
el auth login
|
|
59
|
+
el auth status --check
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
For an existing project API key, use the masked configuration wizard:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
el auth config
|
|
66
|
+
el auth status --check
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
CI can read credentials from `ENVLOOP_API_KEY` and connection settings from
|
|
70
|
+
`ENVLOOP_BASE_URL`, `ENVLOOP_WORKSPACE_ID` and `ENVLOOP_PROJECT_ID`. Keep keys in
|
|
71
|
+
your secret manager. Public API access is scoped to the authorized project.
|
|
72
|
+
|
|
73
|
+
Credentials and settings live in `~/.envloop` (override with an absolute
|
|
74
|
+
`ENVLOOP_HOME`). On POSIX the directory must be private and owned by you. Use the
|
|
75
|
+
default path when the optional environment variable is unset:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
mkdir -p "${ENVLOOP_HOME:-$HOME/.envloop}"
|
|
79
|
+
chmod 700 "${ENVLOOP_HOME:-$HOME/.envloop}"
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
An insecure directory causes `el` to fail with `CONFIG_INSECURE_PATH`.
|
|
83
|
+
|
|
84
|
+
## Explore your project
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
el job list --limit 5
|
|
88
|
+
el job list --type train --limit 5
|
|
89
|
+
el trail list --limit 5
|
|
90
|
+
el trail list --type train --job-id JOB_ID
|
|
91
|
+
el checkpoint list --limit 5
|
|
92
|
+
el job list --type inference
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Query a returned Job ID with `el job get JOB_ID` and `el job result JOB_ID`.
|
|
96
|
+
Trial commands use the spelling `trail`, for example
|
|
97
|
+
`el trail artifact list TRIAL_ID`. Use each command's `--help` for options.
|
|
98
|
+
|
|
99
|
+
For the Python SDK:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from envloop import Client
|
|
103
|
+
|
|
104
|
+
with Client.remote() as client:
|
|
105
|
+
print(client.jobs.list(limit=5))
|
|
106
|
+
print(client.trials.list(type="train", limit=5))
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Submitting work, training a model or starting inference can allocate billable
|
|
110
|
+
resources. Follow the task-specific guide and retain the outputs you need before
|
|
111
|
+
cleaning up temporary resources.
|
|
112
|
+
|
|
113
|
+
## Sandbox jobs
|
|
114
|
+
|
|
115
|
+
With a matching platform and a published `envhub/base` ANS Runner, submit a
|
|
116
|
+
Sandbox Job without a Dataset:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
el job submit --type sandbox --runner base --command 'python --version'
|
|
120
|
+
el job submit --type sandbox --runner base --image python:3.11 --wait
|
|
121
|
+
el job list --type sandbox
|
|
122
|
+
el job log JOB_ID --follow
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
The platform resolves `base` to the registered `envhub/base@main` source ref and
|
|
126
|
+
freezes its immutable Revision. The Sandbox image defaults to `python:3.11`;
|
|
127
|
+
`--image` accepts an image name, tag or digest. The execution provider is configured
|
|
128
|
+
by the server. Job logs retain the actual captured output and collection metadata.
|
|
129
|
+
Failures include a code, readable explanation, HTTP status when available, and
|
|
130
|
+
recovery guidance without echoing arbitrary response bodies or credentials.
|
|
131
|
+
|
|
132
|
+
## Existing installations
|
|
133
|
+
|
|
134
|
+
This distribution provides both the CLI and the `envloop` import package. Use a
|
|
135
|
+
fresh environment if you previously installed the separate source distribution
|
|
136
|
+
named `envloop`, so two distributions do not own the same Python module files.
|
|
137
|
+
Existing client configuration is reused; installing a package does not change
|
|
138
|
+
your account permissions or move remote resources. Check `el --help` when
|
|
139
|
+
upgrading because command groups may differ from your earlier installation.
|
|
140
|
+
|
|
141
|
+
## Documentation
|
|
142
|
+
|
|
143
|
+
- [English documentation](https://docs.envloop.ai/)
|
|
144
|
+
- [中文文档](https://docs.envloop.ai/zh-cn/)
|
|
145
|
+
- [Installation](https://docs.envloop.ai/cli-install)
|
|
146
|
+
- [CLI reference](https://docs.envloop.ai/reference/cli)
|
|
147
|
+
- [Training and deployment](https://docs.envloop.ai/training/train-deploy)
|
|
148
|
+
- [Agent entry: skill.md](https://docs.envloop.ai/skill.md)
|
|
149
|
+
|
|
150
|
+
## Submit managed training
|
|
151
|
+
|
|
152
|
+
Prepare a recipe-supported `REQUEST.json` using the [SFT quickstart](https://docs.envloop.ai/training/quickstart).
|
|
153
|
+
Training uses billable resources. Check your project permissions, balance and data format, then submit once:
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
el job submit --type train --file REQUEST.json
|
|
157
|
+
el job get JOB_ID
|
|
158
|
+
el job log JOB_ID --follow
|
|
159
|
+
el job result JOB_ID
|
|
160
|
+
el checkpoint list --job-id JOB_ID
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
Replace `JOB_ID` with the receipt's value. Read the training status and logs, then find the registered checkpoint after success.
|
|
164
|
+
Ctrl-C ends log reading; to stop training, use `el job stop JOB_ID` and check its state and resource cleanup.
|
|
165
|
+
See [Checkpoints](https://docs.envloop.ai/training/checkpoints) to download or delete saved weights.
|
|
166
|
+
|
|
167
|
+
The built-in SFT recipe does not support custom loss, RL or optimizer-state resume.
|
|
168
|
+
For an older interactive training script, follow the [managed training migration](https://docs.envloop.ai/training/sdk-compatibility).
|
|
169
|
+
|
|
170
|
+
## Run a GPU Runner
|
|
171
|
+
|
|
172
|
+
Choose a published Runner readable in your project. Save this request as `gpu-run.json`, replacing `FULL_COMMIT` with the full Revision from `el runner list --json`:
|
|
173
|
+
|
|
174
|
+
```json
|
|
175
|
+
{
|
|
176
|
+
"request_id": "gpu-run-001",
|
|
177
|
+
"trials": [
|
|
178
|
+
{"name": "first", "spec": {"runner": "envhub/bash-runner@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_sec": 600}},
|
|
179
|
+
{"name": "second", "spec": {"runner": "envhub/bash-runner@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_sec": 600}}
|
|
180
|
+
]
|
|
181
|
+
}
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
el job submit --type train --file gpu-run.json
|
|
186
|
+
el job get JOB_ID
|
|
187
|
+
el job log JOB_ID --follow
|
|
188
|
+
el job result JOB_ID
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Replace `JOB_ID` with the returned value. The example has two independent Trials, each requesting four H100 GPUs: concurrent execution needs eight GPUs. Capacity and budget determine actual concurrency. Startup and execution are billable; check the Runner requirements, project `training.run` permission and [training prices](https://docs.envloop.ai/pricing) first. Reduce the Trial or GPU count when your program supports a smaller first run.
|
|
192
|
+
|
|
193
|
+
The request file does not depend on GPU shortcut flags. If `el job submit --help` lists both `--trails` and `--gput-type`, you can instead use the shortcuts in [GPU Runner training](https://docs.envloop.ai/training/runner). Choose one submission method to avoid duplicate runs. The SDK accepts the same dictionary through `client.training_jobs.submit(request)`.
|
|
194
|
+
|
|
195
|
+
The platform starts the published Revision’s `main.sh` / `main.py` using the Runner image; it does not upload local code. An optional command can only supply a declared `run_command` parameter. If this mode or GPU type is unavailable, contact your project administrator. Use `el job stop JOB_ID` to stop every Trial, then inspect each Trial’s `cleanup_confirmed` and download the uploaded artifacts you need.
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# envloop-cli: EnvLoop CLI and Python SDK
|
|
2
|
+
|
|
3
|
+
Install the public client to run agent Jobs, inspect Trials, manage training and
|
|
4
|
+
checkpoints, and deploy or call inference endpoints. The `el` and `envloop`
|
|
5
|
+
commands are equivalent. The Python import package is `envloop`.
|
|
6
|
+
|
|
7
|
+
## Version 1.7
|
|
8
|
+
|
|
9
|
+
- Use published Runner resource shortcuts for Train Jobs and fixed Checkpoint inputs for Inference Jobs.
|
|
10
|
+
- Pass `--json` explicitly to `job submit`; failures retain safe error codes and recovery guidance.
|
|
11
|
+
- Submit training through Train Jobs. The retired interactive `envloop.training` interface is removed.
|
|
12
|
+
|
|
13
|
+
After upgrading, check `el --version` and `el job submit --help`. Execution also
|
|
14
|
+
requires a matching platform, project permissions and published Runners; package
|
|
15
|
+
installation alone does not establish those prerequisites. See the
|
|
16
|
+
[PyPI release history](https://pypi.org/project/envloop-cli/#history) for published versions.
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
Use a Python virtual environment:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
python -m pip install --upgrade envloop-cli
|
|
24
|
+
el --help
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
The standard installation includes the client dependencies for managed training.
|
|
28
|
+
Check that the SDK can be imported:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
python -c 'from envloop import Client; print("EnvLoop SDK ready")'
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
When you see `EnvLoop SDK ready`, continue to Connect. This check imports the
|
|
35
|
+
client without creating a Job. Submit training through `client.training_jobs`;
|
|
36
|
+
follow the [managed training guide](https://docs.envloop.ai/training/cli-hello-world).
|
|
37
|
+
|
|
38
|
+
Install the client in your Python environment; the platform provides the execution resources.
|
|
39
|
+
|
|
40
|
+
## Connect
|
|
41
|
+
|
|
42
|
+
Use an authorized EnvLoop account and project. Browser authorization:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
el auth login
|
|
46
|
+
el auth status --check
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
For an existing project API key, use the masked configuration wizard:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
el auth config
|
|
53
|
+
el auth status --check
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
CI can read credentials from `ENVLOOP_API_KEY` and connection settings from
|
|
57
|
+
`ENVLOOP_BASE_URL`, `ENVLOOP_WORKSPACE_ID` and `ENVLOOP_PROJECT_ID`. Keep keys in
|
|
58
|
+
your secret manager. Public API access is scoped to the authorized project.
|
|
59
|
+
|
|
60
|
+
Credentials and settings live in `~/.envloop` (override with an absolute
|
|
61
|
+
`ENVLOOP_HOME`). On POSIX the directory must be private and owned by you. Use the
|
|
62
|
+
default path when the optional environment variable is unset:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
mkdir -p "${ENVLOOP_HOME:-$HOME/.envloop}"
|
|
66
|
+
chmod 700 "${ENVLOOP_HOME:-$HOME/.envloop}"
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
An insecure directory causes `el` to fail with `CONFIG_INSECURE_PATH`.
|
|
70
|
+
|
|
71
|
+
## Explore your project
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
el job list --limit 5
|
|
75
|
+
el job list --type train --limit 5
|
|
76
|
+
el trail list --limit 5
|
|
77
|
+
el trail list --type train --job-id JOB_ID
|
|
78
|
+
el checkpoint list --limit 5
|
|
79
|
+
el job list --type inference
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Query a returned Job ID with `el job get JOB_ID` and `el job result JOB_ID`.
|
|
83
|
+
Trial commands use the spelling `trail`, for example
|
|
84
|
+
`el trail artifact list TRIAL_ID`. Use each command's `--help` for options.
|
|
85
|
+
|
|
86
|
+
For the Python SDK:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from envloop import Client
|
|
90
|
+
|
|
91
|
+
with Client.remote() as client:
|
|
92
|
+
print(client.jobs.list(limit=5))
|
|
93
|
+
print(client.trials.list(type="train", limit=5))
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Submitting work, training a model or starting inference can allocate billable
|
|
97
|
+
resources. Follow the task-specific guide and retain the outputs you need before
|
|
98
|
+
cleaning up temporary resources.
|
|
99
|
+
|
|
100
|
+
## Sandbox jobs
|
|
101
|
+
|
|
102
|
+
With a matching platform and a published `envhub/base` ANS Runner, submit a
|
|
103
|
+
Sandbox Job without a Dataset:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
el job submit --type sandbox --runner base --command 'python --version'
|
|
107
|
+
el job submit --type sandbox --runner base --image python:3.11 --wait
|
|
108
|
+
el job list --type sandbox
|
|
109
|
+
el job log JOB_ID --follow
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The platform resolves `base` to the registered `envhub/base@main` source ref and
|
|
113
|
+
freezes its immutable Revision. The Sandbox image defaults to `python:3.11`;
|
|
114
|
+
`--image` accepts an image name, tag or digest. The execution provider is configured
|
|
115
|
+
by the server. Job logs retain the actual captured output and collection metadata.
|
|
116
|
+
Failures include a code, readable explanation, HTTP status when available, and
|
|
117
|
+
recovery guidance without echoing arbitrary response bodies or credentials.
|
|
118
|
+
|
|
119
|
+
## Existing installations
|
|
120
|
+
|
|
121
|
+
This distribution provides both the CLI and the `envloop` import package. Use a
|
|
122
|
+
fresh environment if you previously installed the separate source distribution
|
|
123
|
+
named `envloop`, so two distributions do not own the same Python module files.
|
|
124
|
+
Existing client configuration is reused; installing a package does not change
|
|
125
|
+
your account permissions or move remote resources. Check `el --help` when
|
|
126
|
+
upgrading because command groups may differ from your earlier installation.
|
|
127
|
+
|
|
128
|
+
## Documentation
|
|
129
|
+
|
|
130
|
+
- [English documentation](https://docs.envloop.ai/)
|
|
131
|
+
- [中文文档](https://docs.envloop.ai/zh-cn/)
|
|
132
|
+
- [Installation](https://docs.envloop.ai/cli-install)
|
|
133
|
+
- [CLI reference](https://docs.envloop.ai/reference/cli)
|
|
134
|
+
- [Training and deployment](https://docs.envloop.ai/training/train-deploy)
|
|
135
|
+
- [Agent entry: skill.md](https://docs.envloop.ai/skill.md)
|
|
136
|
+
|
|
137
|
+
## Submit managed training
|
|
138
|
+
|
|
139
|
+
Prepare a recipe-supported `REQUEST.json` using the [SFT quickstart](https://docs.envloop.ai/training/quickstart).
|
|
140
|
+
Training uses billable resources. Check your project permissions, balance and data format, then submit once:
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
el job submit --type train --file REQUEST.json
|
|
144
|
+
el job get JOB_ID
|
|
145
|
+
el job log JOB_ID --follow
|
|
146
|
+
el job result JOB_ID
|
|
147
|
+
el checkpoint list --job-id JOB_ID
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Replace `JOB_ID` with the receipt's value. Read the training status and logs, then find the registered checkpoint after success.
|
|
151
|
+
Ctrl-C ends log reading; to stop training, use `el job stop JOB_ID` and check its state and resource cleanup.
|
|
152
|
+
See [Checkpoints](https://docs.envloop.ai/training/checkpoints) to download or delete saved weights.
|
|
153
|
+
|
|
154
|
+
The built-in SFT recipe does not support custom loss, RL or optimizer-state resume.
|
|
155
|
+
For an older interactive training script, follow the [managed training migration](https://docs.envloop.ai/training/sdk-compatibility).
|
|
156
|
+
|
|
157
|
+
## Run a GPU Runner
|
|
158
|
+
|
|
159
|
+
Choose a published Runner readable in your project. Save this request as `gpu-run.json`, replacing `FULL_COMMIT` with the full Revision from `el runner list --json`:
|
|
160
|
+
|
|
161
|
+
```json
|
|
162
|
+
{
|
|
163
|
+
"request_id": "gpu-run-001",
|
|
164
|
+
"trials": [
|
|
165
|
+
{"name": "first", "spec": {"runner": "envhub/bash-runner@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_sec": 600}},
|
|
166
|
+
{"name": "second", "spec": {"runner": "envhub/bash-runner@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_sec": 600}}
|
|
167
|
+
]
|
|
168
|
+
}
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
el job submit --type train --file gpu-run.json
|
|
173
|
+
el job get JOB_ID
|
|
174
|
+
el job log JOB_ID --follow
|
|
175
|
+
el job result JOB_ID
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Replace `JOB_ID` with the returned value. The example has two independent Trials, each requesting four H100 GPUs: concurrent execution needs eight GPUs. Capacity and budget determine actual concurrency. Startup and execution are billable; check the Runner requirements, project `training.run` permission and [training prices](https://docs.envloop.ai/pricing) first. Reduce the Trial or GPU count when your program supports a smaller first run.
|
|
179
|
+
|
|
180
|
+
The request file does not depend on GPU shortcut flags. If `el job submit --help` lists both `--trails` and `--gput-type`, you can instead use the shortcuts in [GPU Runner training](https://docs.envloop.ai/training/runner). Choose one submission method to avoid duplicate runs. The SDK accepts the same dictionary through `client.training_jobs.submit(request)`.
|
|
181
|
+
|
|
182
|
+
The platform starts the published Revision’s `main.sh` / `main.py` using the Runner image; it does not upload local code. An optional command can only supply a declared `run_command` parameter. If this mode or GPU type is unavailable, contact your project administrator. Use `el job stop JOB_ID` to stop every Trial, then inspect each Trial’s `cleanup_confirmed` and download the uploaded artifacts you need.
|
|
@@ -133,11 +133,11 @@ def log_page_size(value):
|
|
|
133
133
|
|
|
134
134
|
def runner_arguments(submit):
|
|
135
135
|
submit.add_argument("--runner", metavar="NAME[@REVISION]",
|
|
136
|
-
help="Agent:
|
|
136
|
+
help="Agent/Train: namespace/name@REVISION; Sandbox: registered Runner name (for example base)")
|
|
137
137
|
submit.add_argument("--job-id", help="caller-selected Job ID")
|
|
138
138
|
submit.add_argument("--tag", help="run label")
|
|
139
139
|
submit.add_argument("--priority", type=int, help="waiting priority; larger values run first (default: 0)")
|
|
140
|
-
submit.add_argument("--command", help="shell
|
|
140
|
+
submit.add_argument("--command", help="Agent/Sandbox: shell override; Train: declared run_command parameter passed to the published Runner entrypoint; preserves multiline text")
|
|
141
141
|
|
|
142
142
|
|
|
143
143
|
def parser():
|
|
@@ -153,10 +153,20 @@ def parser():
|
|
|
153
153
|
from .auth_cli import add_auth_commands
|
|
154
154
|
|
|
155
155
|
jobs = groups.add_parser("job", help="Submit and manage Agent, Sandbox, Train and Inference Jobs").add_subparsers(dest="action", required=True)
|
|
156
|
-
job_submit = jobs.add_parser("submit", help="Submit an Agent, Sandbox, Train or Inference Job")
|
|
156
|
+
job_submit = jobs.add_parser("submit", help="Submit an Agent, Sandbox, Train or Inference Job", allow_abbrev=False)
|
|
157
157
|
job_submit.add_argument("--type", choices=("agent", "sandbox", "train", "inference"), default="agent", help="Job type (default: agent)")
|
|
158
|
-
job_submit.add_argument("--image", metavar="IMAGE", help="Sandbox
|
|
159
|
-
|
|
158
|
+
job_submit.add_argument("--image", metavar="IMAGE", help="Sandbox image override (server default: python:3.11); Train Runner: public spelling of --docker")
|
|
159
|
+
from .train_runner_cli import memory_mebibytes, positive_integer
|
|
160
|
+
|
|
161
|
+
job_submit.add_argument("--docker", metavar="IMAGE", help="Train Runner Docker image override; omitted uses the Runner declaration")
|
|
162
|
+
job_submit.add_argument("--trails", type=positive_integer, metavar="COUNT", help="Train Runner: 1–16 independent Trials (default: 1)")
|
|
163
|
+
job_submit.add_argument("--gput-type", metavar="MODEL", help="Train Runner: GPU model for each Trial (for example H100)")
|
|
164
|
+
job_submit.add_argument("--gpu", type=positive_integer, metavar="COUNT", help="Train Runner: positive GPU count per Trial")
|
|
165
|
+
job_submit.add_argument("--cpu", type=positive_integer, metavar="COUNT", help="Train Runner: positive CPU count per Trial")
|
|
166
|
+
for option in ('memory', 'disk', 'shm-size'):
|
|
167
|
+
job_submit.add_argument('--' + option, type=memory_mebibytes, metavar="SIZE",
|
|
168
|
+
help="Train Runner: per-Trial size in Mi/Gi/Ti (for example 64Gi)")
|
|
169
|
+
job_submit.add_argument("--file", type=Path, help="Managed Train or Inference Job JSON (spec or Train trials array; interactive training is unsupported)")
|
|
160
170
|
job_submit.add_argument("--source", type=Path, help="two-file directory for a single-task Job")
|
|
161
171
|
job_submit.add_argument("--task-id", help="select one task within the dataset")
|
|
162
172
|
job_submit.add_argument(
|
|
@@ -175,10 +185,11 @@ def parser():
|
|
|
175
185
|
help="named runtime Profile; omitted version requires an explicit default")
|
|
176
186
|
job_submit.add_argument("--seed", type=int)
|
|
177
187
|
job_submit.add_argument("--timeout-sec", type=wait_seconds,
|
|
178
|
-
help="Agent/Sandbox execution timeout (whole seconds); Train wait timeout (default: 600)")
|
|
188
|
+
help="Agent/Sandbox or Train Runner execution timeout (whole seconds); Train --file wait timeout (default: 600)")
|
|
179
189
|
job_submit.add_argument("--delete-delay", type=nonnegative_seconds, metavar="SECONDS",
|
|
180
190
|
help="keep each sandbox (E2B or Fly) after completion for SECONDS (default: 0); cancellation/timeout delete immediately")
|
|
181
191
|
job_submit.add_argument("--wait", action="store_true")
|
|
192
|
+
job_submit.add_argument("--json", action="store_true", help="Output the complete response as JSON (the default for submit; accepted for consistency with list commands)")
|
|
182
193
|
for action in ("get", "stop", "result", "pause", "resume", "wake", "event"):
|
|
183
194
|
command = jobs.add_parser(action)
|
|
184
195
|
command.add_argument("job_id")
|
|
@@ -394,16 +405,27 @@ def main(argv=None):
|
|
|
394
405
|
print(format_error(exc, args=args, program=cli.prog), file=sys.stderr)
|
|
395
406
|
return 1
|
|
396
407
|
if args.group == "job" and args.action == "submit":
|
|
408
|
+
from .train_runner_cli import SHORTCUT_FIELDS, runner_request
|
|
409
|
+
|
|
410
|
+
runner_mode = args.type == 'train' and args.file is None
|
|
411
|
+
if not runner_mode and any(getattr(args, key) is not None for key in SHORTCUT_FIELDS):
|
|
412
|
+
cli.error('Train Runner shortcuts require --type train --runner and cannot be combined with --file')
|
|
397
413
|
if args.type in {"train", "inference"}:
|
|
398
|
-
if args.file is None:
|
|
414
|
+
if args.file is None and not runner_mode:
|
|
399
415
|
cli.error(f"--type {args.type} requires --file REQUEST.json")
|
|
400
416
|
if args.type == 'inference' and (args.wait or args.timeout_sec is not None):
|
|
401
417
|
cli.error('Inference is a continuous service; --wait and --timeout-sec do not apply')
|
|
402
418
|
agent_fields = ("dataset", "dataset_option", "source", "task_id", "config", "split", "revision",
|
|
403
|
-
"
|
|
404
|
-
|
|
419
|
+
"job_id", "tag", "priority", "attempts", "profile", "seed", "delete_delay")
|
|
420
|
+
if not runner_mode:
|
|
421
|
+
agent_fields += ('runner', 'command', 'request_id', 'image')
|
|
405
422
|
if any(getattr(args, key) is not None for key in agent_fields):
|
|
406
|
-
cli.error(f"--type {args.type}
|
|
423
|
+
cli.error(f"--type {args.type} cannot be combined with Agent submission options or mixed --file/Runner input")
|
|
424
|
+
if runner_mode:
|
|
425
|
+
try:
|
|
426
|
+
args.train_request = runner_request(args)
|
|
427
|
+
except ValueError as exc:
|
|
428
|
+
cli.error(str(exc))
|
|
407
429
|
elif args.type == 'sandbox':
|
|
408
430
|
from .sandbox import sandbox_runner_selector, validate_image
|
|
409
431
|
|
|
@@ -506,14 +528,15 @@ def main(argv=None):
|
|
|
506
528
|
if args.type == 'inference':
|
|
507
529
|
result = client.jobs.submit_inference(json.loads(args.file.read_text(encoding='utf-8')))
|
|
508
530
|
elif args.type == "train":
|
|
509
|
-
|
|
531
|
+
request = args.train_request if args.file is None else json.loads(args.file.read_text(encoding="utf-8"))
|
|
532
|
+
result = client.training_jobs.submit(request)
|
|
510
533
|
else:
|
|
511
534
|
result = client.jobs.submit(args.dataset, type=args.type, source=args.source, task_id=args.task_id,
|
|
512
535
|
attempts=args.attempts if args.attempts is not None else 1,
|
|
513
536
|
request_id=args.request_id, config=args.config, split=args.split,
|
|
514
537
|
revision=args.revision, **submission_options(args))
|
|
515
538
|
if args.wait:
|
|
516
|
-
wait_options = {"timeout": args.timeout_sec} if args.type == "train" and args.timeout_sec is not None else {}
|
|
539
|
+
wait_options = {"timeout": args.timeout_sec} if args.type == "train" and args.file is not None and args.timeout_sec is not None else {}
|
|
517
540
|
result = client.jobs.wait(result["job_id"], **wait_options)
|
|
518
541
|
elif args.action == 'event':
|
|
519
542
|
result = client.jobs.events(args.job_id)
|
|
@@ -90,6 +90,11 @@ _register("RSI_CONTROL_UNCONFIGURED RSI_CONTROL_UNAVAILABLE RSI_CONTROL_TIMEOUT"
|
|
|
90
90
|
_register("JOB_ACTION_UNSUPPORTED", "This Job type does not support the requested action.", "Check the Job type with '{program} job get JOB_ID' and use its supported actions.")
|
|
91
91
|
_register("INFERENCE_ROUTE_RETIRED", "The global inference route has been retired.", "Use the owning Job's base URL: API_BASE/v1/job/JOB_ID/Inference, and a model deployed by that Job.")
|
|
92
92
|
|
|
93
|
+
_register("CHECKPOINT_UNAVAILABLE CHECKPOINT_NOT_DURABLE CHECKPOINT_FILES_MISSING", "The fixed inference input is unavailable or not durable.", "Inspect checkpoint publication, expiry, deletion and permissions. Do not retry with different weights under the same request ID.")
|
|
94
|
+
_register("CHECKPOINT_INTEGRITY_FAILED CHECKPOINT_FORMAT_UNSUPPORTED CHECKPOINT_BASE_MISMATCH", "The fixed input failed integrity or compatibility validation.", "Use a verified durable checkpoint matching the serving profile. A different input requires a new Job and request ID.")
|
|
95
|
+
_register("INFERENCE_AUTHORIZATION_DENIED INFERENCE_AUTHORIZATION_UNAVAILABLE", "The platform could not authorize a new inference worker.", "Check current project membership and Runner/input access before waking or resuming the Job.")
|
|
96
|
+
_register("INFERENCE_PROFILE_UNSUPPORTED INFERENCE_RUNNER_INVALID INFERENCE_MODEL_UNPINNED INFERENCE_RUNNER_IMAGE_MISMATCH INFERENCE_RUNNER_IMAGE_UNAVAILABLE INFERENCE_RUNTIME_UNAVAILABLE", "The selected Runner or frozen inference configuration is unavailable or unsupported.", "Discover a published compatible Runner and serving profile. Submit a new Job for changes to weights, base model or Runner.")
|
|
97
|
+
|
|
93
98
|
# Public service identifiers verified against EnvPlatform. Keep unexpected codes
|
|
94
99
|
# private even when they look uppercase: a server can echo a submitted secret in
|
|
95
100
|
# that field. HTTP status still supplies useful recovery for newer service codes.
|
|
@@ -548,10 +548,8 @@ class Jobs:
|
|
|
548
548
|
|
|
549
549
|
def submit_inference(self, body: dict):
|
|
550
550
|
"""Submit a continuous inference deployment, identified by its Job ID."""
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
if not isinstance(body.get('spec'), dict):
|
|
554
|
-
raise ValueError('inference submission requires a spec object')
|
|
551
|
+
from .inference import validate_submission
|
|
552
|
+
validate_submission(body)
|
|
555
553
|
return self._server.submit_job({'request_id': str(uuid4()), **body, 'type': 'inference'})
|
|
556
554
|
|
|
557
555
|
def pause(self, job_id):
|
|
@@ -646,36 +644,11 @@ class Client:
|
|
|
646
644
|
|
|
647
645
|
return TemporaryResources(self, **options)
|
|
648
646
|
|
|
649
|
-
def training_service(self,
|
|
650
|
-
"""
|
|
647
|
+
def training_service(self, *args, **options):
|
|
648
|
+
"""Removed: raise a pointer to managed Jobs."""
|
|
649
|
+
from .train import INTERACTIVE_TRAINING_REMOVED
|
|
651
650
|
|
|
652
|
-
|
|
653
|
-
token. Upstream token refresh follows the same platform URL and the
|
|
654
|
-
gateway rechecks the underlying subject and credential on every request.
|
|
655
|
-
"""
|
|
656
|
-
from .training import ServiceClient
|
|
657
|
-
from .training.credentials import TrainingHttpClient
|
|
658
|
-
|
|
659
|
-
if 'http_client' in options:
|
|
660
|
-
raise ValueError('Client.training_service manages its authenticated HTTP client')
|
|
661
|
-
if any(name in options for name in ('base_url', 'api_key', 'default_headers')):
|
|
662
|
-
raise ValueError('Client.training_service manages its connection and credential headers')
|
|
663
|
-
overrides = options.get('default_headers_overrides') or {}
|
|
664
|
-
protected_headers = {'authorization', 'x-api-key', 'cookie', 'x-workspace-id',
|
|
665
|
-
'x-project-id', 'x-envloop-temporary-scope'}
|
|
666
|
-
if any(name.lower() in protected_headers for name in overrides):
|
|
667
|
-
raise ValueError('Client.training_service manages authentication, project and temporary scope headers')
|
|
668
|
-
transport = self._transport
|
|
669
|
-
project_id = transport.headers.get("X-Project-ID", "")
|
|
670
|
-
if 'project_id' in options and options.pop('project_id') != project_id:
|
|
671
|
-
raise ValueError('Select the training project through Client.remote(project_id=...)')
|
|
672
|
-
token = transport.request("POST", "/v1/job/tinker/api/v1/auth/token", {})["jwt"]
|
|
673
|
-
headers = {name: value for name, value in transport.headers.items() if name in {"X-Workspace-ID", "X-Project-ID"}}
|
|
674
|
-
if temporary_scope_id is not None:
|
|
675
|
-
headers["X-EnvLoop-Temporary-Scope"] = temporary_scope_id
|
|
676
|
-
base_url = transport.base + "/v1/job/tinker"
|
|
677
|
-
return ServiceClient(base_url=base_url, api_key=token, project_id=project_id, default_headers=headers,
|
|
678
|
-
http_client=TrainingHttpClient(base_url, transport.headers["Authorization"]), **options)
|
|
651
|
+
raise RuntimeError(INTERACTIVE_TRAINING_REMOVED)
|
|
679
652
|
|
|
680
653
|
@classmethod
|
|
681
654
|
def remote(cls, api_url=None, *, timeout=30, api_key=None, workspace_id=None, project_id=None, config=True):
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Public inference intent validation; resolution and admission remain server owned."""
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
_FIELDS = {'name', 'runner', 'inputs', 'serving_profile', 'hardware', 'weights_format',
|
|
5
|
+
'scaling', 'temporary_scope_id'}
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def validate_submission(body):
|
|
9
|
+
if (not isinstance(body, dict) or set(body) - {'type', 'request_id', 'spec'}
|
|
10
|
+
or body.get('type', 'inference') != 'inference'):
|
|
11
|
+
raise ValueError('inference submission requires spec and request_id')
|
|
12
|
+
spec = body.get('spec')
|
|
13
|
+
if not isinstance(spec, dict):
|
|
14
|
+
raise ValueError('inference submission requires a spec object')
|
|
15
|
+
if 'request_id' in body and (not isinstance(body['request_id'], str)
|
|
16
|
+
or not body['request_id'].strip() or len(body['request_id']) > 128):
|
|
17
|
+
raise ValueError('request_id must be a non-empty string of at most 128 characters')
|
|
18
|
+
if any(key in spec for key in ('provider', 'executor', 'resolved_runner', 'runtime')):
|
|
19
|
+
raise ValueError('execution configuration and Runner resolution are owned by the server')
|
|
20
|
+
if 'runner' not in spec:
|
|
21
|
+
return # Explicit legacy compatibility: existing adapter specs still reach the API.
|
|
22
|
+
if set(spec) - _FIELDS:
|
|
23
|
+
raise ValueError('fixed-input inference cannot mix legacy or runtime fields')
|
|
24
|
+
runner = spec['runner']
|
|
25
|
+
if (not isinstance(runner, str)
|
|
26
|
+
or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*@[^\s@]+', runner)):
|
|
27
|
+
raise ValueError('runner requires namespace/name@revision')
|
|
28
|
+
if any(not isinstance(spec.get(key), str) or not spec[key].strip()
|
|
29
|
+
for key in ('name', 'serving_profile')):
|
|
30
|
+
raise ValueError('fixed-input inference requires name and serving_profile')
|
|
31
|
+
inputs = spec.get('inputs', {})
|
|
32
|
+
if (not isinstance(inputs, dict) or set(inputs) - {'checkpoint'}
|
|
33
|
+
or 'checkpoint' in inputs and (not isinstance(inputs['checkpoint'], str)
|
|
34
|
+
or not inputs['checkpoint'].startswith('envloop://') or not inputs['checkpoint'][10:])):
|
|
35
|
+
raise ValueError('inputs accepts one Checkpoint ANS URI')
|
|
@@ -183,10 +183,6 @@ class TemporaryResources:
|
|
|
183
183
|
def get(self):
|
|
184
184
|
return self.client._transport.request("GET", self.path)
|
|
185
185
|
|
|
186
|
-
def training_service(self, **options):
|
|
187
|
-
self.path # Fail before any authentication/session if the scope was not entered.
|
|
188
|
-
return self.client.training_service(temporary_scope_id=self.scope_id, **options)
|
|
189
|
-
|
|
190
186
|
def retain(self, resources):
|
|
191
187
|
return self.client._transport.request("POST", self.path + "/retain", {"resources": resources})
|
|
192
188
|
|
|
@@ -3,19 +3,42 @@ import math
|
|
|
3
3
|
import time
|
|
4
4
|
from urllib.parse import quote, urlencode
|
|
5
5
|
|
|
6
|
+
from .http import RemoteError
|
|
7
|
+
|
|
8
|
+
INTERACTIVE_TRAINING_REMOVED = (
|
|
9
|
+
"Interactive training (envloop.training, Client.training_service) was removed; training is "
|
|
10
|
+
"managed Jobs only. Submit with `el job submit --type train --file REQUEST.json` "
|
|
11
|
+
"(or client.training_jobs.submit(request)) and discover recipes with "
|
|
12
|
+
"client.training_jobs.models(). See docs/training.md."
|
|
13
|
+
)
|
|
14
|
+
|
|
6
15
|
|
|
7
16
|
class TrainJobs:
|
|
8
17
|
def __init__(self, transport):
|
|
9
18
|
self.transport = transport
|
|
10
19
|
|
|
20
|
+
def models(self) -> dict:
|
|
21
|
+
"""Discover configured managed recipes; admission and GPU readiness remain separate."""
|
|
22
|
+
try:
|
|
23
|
+
return self.transport.request('GET', '/v1/job/training-models')
|
|
24
|
+
except RemoteError as error:
|
|
25
|
+
if error.status_code == 404:
|
|
26
|
+
raise RuntimeError(
|
|
27
|
+
"This platform does not serve /v1/job/training-models; upgrade the platform to a "
|
|
28
|
+
"version that matches this client (managed training discovery is unavailable).") from error
|
|
29
|
+
raise
|
|
30
|
+
|
|
11
31
|
@staticmethod
|
|
12
32
|
def _view(result):
|
|
13
33
|
return {**(result.get('training') or {}), **result}
|
|
14
34
|
|
|
15
35
|
def submit(self, request: dict) -> dict:
|
|
16
|
-
"""Submit one
|
|
36
|
+
"""Submit one managed spec or ``{request_id, trials: [{name?, spec}, ...]}``.
|
|
17
37
|
|
|
18
|
-
Each Trial spec is independent
|
|
38
|
+
Each Trial spec is an independent managed recipe or published Runner
|
|
39
|
+
(``{runner: "namespace/name@REVISION", resources?: {...}, ...}``).
|
|
40
|
+
Runner ``command`` is the declared run_command parameter, never an
|
|
41
|
+
entrypoint override. Validation, defaults and execution selection belong
|
|
19
42
|
to the platform. The training view retains the server's complete response.
|
|
20
43
|
"""
|
|
21
44
|
if not isinstance(request, dict) or request.get("type", "train") != "train":
|