envloop-cli-experiment 1.23.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- envloop_cli_experiment-1.23.0.dist-info/METADATA +11 -0
- envloop_cli_experiment-1.23.0.dist-info/RECORD +23 -0
- envloop_cli_experiment-1.23.0.dist-info/WHEEL +4 -0
- envloop_experiment/__init__.py +1 -0
- envloop_experiment/cli.py +467 -0
- envloop_experiment/cli_output.py +50 -0
- envloop_experiment/client.py +471 -0
- envloop_experiment/decision.py +20 -0
- envloop_experiment/http.py +142 -0
- envloop_experiment/inference.py +36 -0
- envloop_experiment/job_cost.py +41 -0
- envloop_experiment/job_logs.py +21 -0
- envloop_experiment/job_spec_cli.py +146 -0
- envloop_experiment/navigation_resources.py +43 -0
- envloop_experiment/resource_cli.py +33 -0
- envloop_experiment/resources.py +152 -0
- envloop_experiment/runtime_params.py +83 -0
- envloop_experiment/sandbox.py +28 -0
- envloop_experiment/train.py +93 -0
- envloop_experiment/train_log_cli.py +130 -0
- envloop_experiment/train_progress.py +52 -0
- envloop_experiment/train_runner_cli.py +66 -0
- envloop_experiment/training/__init__.py +4 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""One-line view of an Agent/Sandbox Job's ``cost`` and budget; reading never changes the Job."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _usd(value: object) -> str:
|
|
8
|
+
if not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value):
|
|
9
|
+
return "unknown"
|
|
10
|
+
if 0 < value < 0.01:
|
|
11
|
+
# Sub-cent amounts keep their significant digits down to one micro-USD (the platform's unit).
|
|
12
|
+
exact = f"{value:.6f}".rstrip("0")
|
|
13
|
+
return "$" + (exact if exact != "0." else f"{value:.1e}")
|
|
14
|
+
return f"${value:.2f}"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def cost_line(job: Any) -> str | None:
|
|
18
|
+
"""``Cost (estimate): $5.03 (model $4.12, sandbox $0.91) of a $5.00 budget; stopped: budget reached``.
|
|
19
|
+
|
|
20
|
+
None when the response has no ``cost`` block (Train and Inference Jobs, older platforms). An unknown
|
|
21
|
+
part is shown as unknown, never as zero.
|
|
22
|
+
"""
|
|
23
|
+
cost = job.get("cost") if isinstance(job, dict) else None
|
|
24
|
+
if not isinstance(cost, dict):
|
|
25
|
+
return None
|
|
26
|
+
line = f"Cost (estimate): {_usd(cost.get('total_usd'))} (model {_usd(cost.get('model_usd'))}, " \
|
|
27
|
+
f"sandbox {_usd(cost.get('sandbox_usd'))})"
|
|
28
|
+
if cost.get("cap_usd") is not None:
|
|
29
|
+
line += f" of a {_usd(cost.get('cap_usd'))} budget"
|
|
30
|
+
if cost.get("unmetered") is True:
|
|
31
|
+
line += "; some usage has no price, so the budget cannot be enforced"
|
|
32
|
+
elif cost.get("budget_exceeded") is True:
|
|
33
|
+
line += "; stopped: budget reached"
|
|
34
|
+
else:
|
|
35
|
+
reserved = cost.get("reserved_usd")
|
|
36
|
+
line += f", {_usd(cost.get('remaining_usd'))} left"
|
|
37
|
+
if isinstance(reserved, (int, float)) and not isinstance(reserved, bool) and reserved > 0:
|
|
38
|
+
line += f" ({_usd(reserved)} reserved by running Trials)"
|
|
39
|
+
if cost.get("billing_mode") in {"off", "shadow"}:
|
|
40
|
+
line += f"; sandbox time not charged ({cost['billing_mode']} billing)"
|
|
41
|
+
return line
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Selection shared by generic and training Job log views."""
|
|
2
|
+
import re
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def log_query(*, source: str = 'all', cursor: str | None = None, tail: int | None = None, limit: int = 200, process: str | None = None) -> dict[str, Any]:
|
|
7
|
+
if source not in {'all', 'system', 'runtime'}:
|
|
8
|
+
raise ValueError('source must be all, system or runtime')
|
|
9
|
+
# One recorded process, for example ``trainer`` (the trainer's step metrics) or ``controller``.
|
|
10
|
+
if process is not None and (not isinstance(process, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', process)):
|
|
11
|
+
raise ValueError('process must be 1-128 letters, digits, ".", "_", ":" or "-"')
|
|
12
|
+
for name, value in (('limit', limit), ('tail', tail)):
|
|
13
|
+
if (name == 'limit' or value is not None) and (type(value) is not int or not 1 <= value <= 1000):
|
|
14
|
+
raise ValueError(f'{name} must be an integer from 1 to 1000')
|
|
15
|
+
if cursor is not None and (not isinstance(cursor, str) or not cursor):
|
|
16
|
+
raise ValueError('cursor must be a non-empty string')
|
|
17
|
+
if cursor is not None and tail is not None:
|
|
18
|
+
raise ValueError('cursor and tail are mutually exclusive')
|
|
19
|
+
return {key: value for key, value in {
|
|
20
|
+
'source': source, 'process': process, 'cursor': cursor, 'tail': tail, 'limit': limit,
|
|
21
|
+
}.items() if value is not None}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Merge explicit Job CLI inputs into the public, type-specific request."""
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
from collections.abc import Iterable, Mapping
|
|
5
|
+
from copy import deepcopy
|
|
6
|
+
from types import SimpleNamespace
|
|
7
|
+
from typing import Any, cast
|
|
8
|
+
|
|
9
|
+
from envloop_experiment.client import dataset_selector, runner_selector
|
|
10
|
+
from envloop_experiment.train_runner_cli import SHORTCUT_FIELDS, runner_request
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class SpecFileError(ValueError):
|
|
14
|
+
"""The input file is not a Job JSON object."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def spec_request(args: argparse.Namespace):
|
|
18
|
+
if args.trails is not None:
|
|
19
|
+
raise ValueError('--trails cannot be combined with --spec; the file defines its Trial list')
|
|
20
|
+
if args.type in {'agent', 'sandbox', 'inference'} and any(getattr(args, key) is not None for key in SHORTCUT_FIELDS):
|
|
21
|
+
raise ValueError('Train Runner shortcuts require --type train')
|
|
22
|
+
if args.type == 'inference':
|
|
23
|
+
if args.wait or args.timeout_sec is not None:
|
|
24
|
+
raise ValueError('Inference is a continuous service; --wait and --timeout-sec do not apply')
|
|
25
|
+
if args.runner is not None:
|
|
26
|
+
runner_selector(args.runner)
|
|
27
|
+
request = json.loads(args.spec.read_text(encoding='utf-8'))
|
|
28
|
+
if not isinstance(request, dict):
|
|
29
|
+
raise SpecFileError('Job spec must be a JSON object')
|
|
30
|
+
request = deepcopy(request)
|
|
31
|
+
kind = args.type or request.get('type', 'agent')
|
|
32
|
+
if kind not in {'agent', 'sandbox', 'train', 'inference'}:
|
|
33
|
+
raise ValueError('unsupported Job spec type')
|
|
34
|
+
args.type = kind
|
|
35
|
+
request['type'] = kind
|
|
36
|
+
if args.request_id is not None:
|
|
37
|
+
request['request_id'] = args.request_id
|
|
38
|
+
if args.trails is not None:
|
|
39
|
+
raise ValueError('--trails cannot be combined with --spec; the file defines its Trial list')
|
|
40
|
+
if args.dataset is not None and args.dataset_option is not None:
|
|
41
|
+
raise ValueError('provide only one positional dataset or --dataset')
|
|
42
|
+
def supplied(names: Iterable[str]):
|
|
43
|
+
return [name for name in names if getattr(args, name) is not None]
|
|
44
|
+
if kind in {'train', 'inference'}:
|
|
45
|
+
invalid = supplied(('dataset', 'dataset_option', 'source', 'task_id', 'split', 'revision',
|
|
46
|
+
'job_id', 'tag', 'attempts', 'profile', 'delete_delay', 'max_cost_usd'))
|
|
47
|
+
if kind == 'inference':
|
|
48
|
+
invalid += supplied(('config', 'priority', 'seed', 'command', 'image', 'timeout_sec', *SHORTCUT_FIELDS))
|
|
49
|
+
if args.wait:
|
|
50
|
+
invalid.append('wait')
|
|
51
|
+
if invalid:
|
|
52
|
+
raise ValueError(f'--type {kind} does not support '+', '.join('--'+name.replace('_', '-') for name in invalid))
|
|
53
|
+
if kind == 'inference':
|
|
54
|
+
if args.runner is not None:
|
|
55
|
+
if not isinstance(request.get('spec'), dict):
|
|
56
|
+
raise ValueError('Inference Job spec requires a spec object')
|
|
57
|
+
request['spec']['runner'] = runner_selector(args.runner)
|
|
58
|
+
return request
|
|
59
|
+
grouped = 'trials' in request or 'spec' in request
|
|
60
|
+
trials = request.get('trials', [{'spec': request.get('spec')}]) if grouped else [{'spec': request}]
|
|
61
|
+
if not isinstance(trials, list) or not trials or any(not isinstance(t, dict) or not isinstance(t.get('spec'), dict) for t in trials):
|
|
62
|
+
raise ValueError('Train Job spec requires non-empty trials with spec objects')
|
|
63
|
+
if args.priority is not None:
|
|
64
|
+
request['priority'] = args.priority
|
|
65
|
+
runner_fields = ('runner', 'command', 'image', 'timeout_sec', *SHORTCUT_FIELDS)
|
|
66
|
+
for trial in trials:
|
|
67
|
+
spec = trial['spec']
|
|
68
|
+
for field in ('seed', 'config'):
|
|
69
|
+
if getattr(args, field) is not None:
|
|
70
|
+
spec[field] = getattr(args, field)
|
|
71
|
+
active_fields = runner_fields
|
|
72
|
+
values = vars(args).copy()
|
|
73
|
+
if not (args.runner or spec.get('runner')) and args.wait:
|
|
74
|
+
active_fields = tuple(field for field in runner_fields if field != 'timeout_sec')
|
|
75
|
+
values['timeout_sec'] = None
|
|
76
|
+
if supplied(active_fields):
|
|
77
|
+
values['runner'] = args.runner if args.runner is not None else spec.get('runner')
|
|
78
|
+
overlay = runner_request(cast(argparse.Namespace, SimpleNamespace(**values)))['trials'][0]['spec']
|
|
79
|
+
if 'resources' in overlay:
|
|
80
|
+
overlay['resources'] = {**spec.get('resources', {}), **overlay['resources']}
|
|
81
|
+
spec.update(overlay)
|
|
82
|
+
return request
|
|
83
|
+
if supplied(SHORTCUT_FIELDS):
|
|
84
|
+
raise ValueError('Train Runner shortcuts require --type train')
|
|
85
|
+
if kind == 'sandbox' and supplied(('dataset', 'dataset_option', 'source', 'task_id', 'config', 'split', 'revision', 'seed')):
|
|
86
|
+
raise ValueError('Sandbox Jobs do not accept Dataset, Task or seed options')
|
|
87
|
+
if kind == 'agent' and args.image is not None:
|
|
88
|
+
raise ValueError('--image requires --type sandbox')
|
|
89
|
+
for field in ('job_id', 'tag', 'priority', 'attempts', 'command', 'image'):
|
|
90
|
+
if getattr(args, field) is not None:
|
|
91
|
+
request[field] = getattr(args, field)
|
|
92
|
+
if args.runner is not None:
|
|
93
|
+
if kind == 'sandbox':
|
|
94
|
+
from envloop_experiment.sandbox import sandbox_runner_selector
|
|
95
|
+
request['runner'] = sandbox_runner_selector(args.runner)
|
|
96
|
+
else:
|
|
97
|
+
request['runner'] = runner_selector(args.runner)
|
|
98
|
+
dataset = args.dataset if args.dataset is not None else args.dataset_option
|
|
99
|
+
if args.source is not None and dataset is not None:
|
|
100
|
+
raise ValueError('provide only one dataset or --source')
|
|
101
|
+
if args.source is not None or dataset is not None:
|
|
102
|
+
for key in ('dataset', 'dataset_ref', 'task', 'task_ref'):
|
|
103
|
+
request.pop(key, None)
|
|
104
|
+
if args.source is not None:
|
|
105
|
+
from envloop_experiment.client import load_task
|
|
106
|
+
request['task'] = load_task(args.source)
|
|
107
|
+
else:
|
|
108
|
+
request['dataset'] = dataset_selector(cast(Mapping[str, Any] | str, dataset))
|
|
109
|
+
selection = {key: getattr(args, key) for key in ('task_id', 'config', 'split', 'revision') if getattr(args, key) is not None}
|
|
110
|
+
if selection:
|
|
111
|
+
base = request.get('dataset', {})
|
|
112
|
+
if isinstance(base, str):
|
|
113
|
+
base = dataset_selector(base)
|
|
114
|
+
if not isinstance(base, dict) or 'task' in request or 'task_ref' in request or 'dataset_ref' in request:
|
|
115
|
+
raise ValueError('dataset selection options require a dataset selector')
|
|
116
|
+
request['dataset'] = {**base, **selection}
|
|
117
|
+
if args.profile is not None:
|
|
118
|
+
for key in ('profile_id', 'profile_name', 'profile_version', 'profile_ref'):
|
|
119
|
+
request.pop(key, None)
|
|
120
|
+
if request.get('runner') is not None:
|
|
121
|
+
request['profile_id'] = args.profile
|
|
122
|
+
else:
|
|
123
|
+
name, _, version = args.profile.partition('@')
|
|
124
|
+
request['profile_name'] = name
|
|
125
|
+
if version:
|
|
126
|
+
request['profile_version'] = version
|
|
127
|
+
if args.seed is not None:
|
|
128
|
+
if request.get('runner') is not None:
|
|
129
|
+
request['profile'] = {**request.get('profile', {}), 'seed': args.seed}
|
|
130
|
+
else:
|
|
131
|
+
request['seed'] = args.seed
|
|
132
|
+
execution = {}
|
|
133
|
+
if args.timeout_sec is not None:
|
|
134
|
+
if not args.timeout_sec.is_integer():
|
|
135
|
+
raise ValueError('--timeout-sec must be whole seconds')
|
|
136
|
+
execution['timeout_sec'] = int(args.timeout_sec)
|
|
137
|
+
if args.delete_delay is not None:
|
|
138
|
+
execution['delete_delay_sec'] = args.delete_delay
|
|
139
|
+
if execution:
|
|
140
|
+
request['execution'] = {**request.get('execution', {}), **execution}
|
|
141
|
+
if args.max_cost_usd is not None:
|
|
142
|
+
request['budget'] = {'max_cost_usd': args.max_cost_usd}
|
|
143
|
+
from envloop_experiment.runtime_params import merge_params
|
|
144
|
+
if 'params' in request or getattr(args, 'params', None) is not None:
|
|
145
|
+
request['params'] = merge_params(request.get('params', {}), getattr(args, 'params', None) or {})
|
|
146
|
+
return request
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
from urllib.parse import quote
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class InferenceResource:
|
|
6
|
+
def __init__(self, transport: Any, path: str):
|
|
7
|
+
self.transport, self.path = transport, path
|
|
8
|
+
|
|
9
|
+
def list(self):
|
|
10
|
+
return self.transport.request('GET', self.path)
|
|
11
|
+
|
|
12
|
+
def get(self, identifier: str):
|
|
13
|
+
return self.transport.request('GET', self.path + '/' + quote(identifier, safe=''))
|
|
14
|
+
|
|
15
|
+
def create(self, body: dict[str, Any]):
|
|
16
|
+
if not isinstance(body, dict):
|
|
17
|
+
raise ValueError('request must be an object')
|
|
18
|
+
return self.transport.request('POST', self.path, body)
|
|
19
|
+
|
|
20
|
+
def action(self, identifier: str, action: str):
|
|
21
|
+
if action not in {'events', 'wake', 'pause', 'resume', 'delete'}:
|
|
22
|
+
raise ValueError('unsupported endpoint action')
|
|
23
|
+
path = self.path + '/' + quote(identifier, safe='')
|
|
24
|
+
method = 'DELETE' if action == 'delete' else 'GET' if action == 'events' else 'POST'
|
|
25
|
+
return self.transport.request(method, path if action == 'delete' else path + '/' + action)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Inference:
|
|
29
|
+
def __init__(self, transport: Any):
|
|
30
|
+
self.profiles = InferenceResource(transport, '/v1/serving/profiles')
|
|
31
|
+
self.adapters = InferenceResource(transport, '/v1/adapters')
|
|
32
|
+
self.endpoints = InferenceResource(transport, '/v1/endpoints')
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Dashboard:
|
|
36
|
+
def __init__(self, project: Any, jobs: Any, training: Any):
|
|
37
|
+
self.project, self.jobs, self.training = project, jobs, training
|
|
38
|
+
|
|
39
|
+
def get(self):
|
|
40
|
+
identity = self.project.transport.request('GET', '/v1/me')
|
|
41
|
+
project = identity.get('project') or {key: identity.get(key) for key in ('workspace_id', 'project_id')}
|
|
42
|
+
return {'project': project, 'jobs': self.jobs.list(limit=25, type='agent'),
|
|
43
|
+
'training': self.training.list(limit=25), 'coverage': 'first_page'}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from envloop.resource_cli import listing
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def add_job_resources(jobs: "argparse._SubParsersAction[argparse.ArgumentParser]"):
|
|
10
|
+
profiles = jobs.add_parser('serving-profile', help='Discover deployable serving profiles').add_subparsers(dest='resource_action', required=True)
|
|
11
|
+
listing(profiles, paged=False)
|
|
12
|
+
adapters = jobs.add_parser('adapter', help='Manage reusable inference adapters').add_subparsers(dest='resource_action', required=True)
|
|
13
|
+
listing(adapters, paged=False)
|
|
14
|
+
adapters.add_parser('get', help='Show an adapter').add_argument('identifier')
|
|
15
|
+
register = adapters.add_parser('register', help='Register an adapter using a JSON request')
|
|
16
|
+
register.add_argument('--file', type=Path, required=True)
|
|
17
|
+
delete = adapters.add_parser('delete', help='Delete an unreferenced adapter')
|
|
18
|
+
delete.add_argument('identifier')
|
|
19
|
+
from envloop.lifecycle_cli import wait_options
|
|
20
|
+
wait_options(delete)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def run_job_resource(client: Any, args: argparse.Namespace):
|
|
24
|
+
resource = client.inference.profiles if args.action == 'serving-profile' else client.inference.adapters
|
|
25
|
+
action = args.resource_action
|
|
26
|
+
if action == 'list':
|
|
27
|
+
return resource.list()
|
|
28
|
+
if action == 'get':
|
|
29
|
+
return resource.get(args.identifier)
|
|
30
|
+
if action == 'register':
|
|
31
|
+
return resource.create(json.loads(args.file.read_text(encoding='utf-8')))
|
|
32
|
+
result = resource.action(args.identifier, 'delete')
|
|
33
|
+
return client.operations.wait(result['operation_id'], timeout=args.timeout_sec) if args.wait else result
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from collections.abc import Sequence
|
|
5
|
+
from types import TracebackType
|
|
6
|
+
from typing import Any
|
|
7
|
+
from uuid import uuid4
|
|
8
|
+
|
|
9
|
+
from envloop.http import RemoteError
|
|
10
|
+
from envloop.resources import CleanupError, _id, _wait_options
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Adapters:
|
|
14
|
+
def __init__(self, transport: Any, temporary_scope_id: str | None = None):
|
|
15
|
+
self.transport, self.temporary_scope_id = transport, temporary_scope_id
|
|
16
|
+
|
|
17
|
+
def list(self):
|
|
18
|
+
return self.transport.request("GET", "/v1/adapters")
|
|
19
|
+
|
|
20
|
+
def get(self, adapter_id: str):
|
|
21
|
+
return self.transport.request("GET", f"/v1/adapters/{_id(adapter_id)}")
|
|
22
|
+
|
|
23
|
+
def register(self, checkpoint: str, *, serving_profile: str, request_id: str | None = None):
|
|
24
|
+
body = {"request_id": request_id or str(uuid4()), "serving_profile": serving_profile,
|
|
25
|
+
"source": {"type": "train_checkpoint", "tinker_path": checkpoint}}
|
|
26
|
+
if self.temporary_scope_id:
|
|
27
|
+
body["temporary_scope_id"] = self.temporary_scope_id
|
|
28
|
+
return self.transport.request("POST", "/v1/adapters", body)
|
|
29
|
+
|
|
30
|
+
def delete(self, adapter_id: str):
|
|
31
|
+
return self.transport.request("DELETE", f"/v1/adapters/{_id(adapter_id)}")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Endpoints:
|
|
35
|
+
def __init__(self, transport: Any, temporary_scope_id: str | None = None):
|
|
36
|
+
self.transport, self.temporary_scope_id = transport, temporary_scope_id
|
|
37
|
+
|
|
38
|
+
def profiles(self):
|
|
39
|
+
return self.transport.request("GET", "/v1/serving/profiles")
|
|
40
|
+
|
|
41
|
+
def list(self):
|
|
42
|
+
return self.transport.request("GET", "/v1/endpoints")
|
|
43
|
+
|
|
44
|
+
def get(self, endpoint_id: str):
|
|
45
|
+
return self.transport.request("GET", f"/v1/endpoints/{_id(endpoint_id)}")
|
|
46
|
+
|
|
47
|
+
def create(self, *, name: str, serving_profile: str, adapters: Sequence[dict[str, str]], request_id: str | None = None, **options: Any):
|
|
48
|
+
body = {"request_id": request_id or str(uuid4()), "name": name, "serving_profile": serving_profile,
|
|
49
|
+
"adapters": adapters, "scaling": {"min_replicas": 0, "max_replicas": 1, "scale_to_zero_after_s": 300},
|
|
50
|
+
**options}
|
|
51
|
+
if self.temporary_scope_id:
|
|
52
|
+
body["temporary_scope_id"] = self.temporary_scope_id
|
|
53
|
+
return self.transport.request("POST", "/v1/endpoints", body)
|
|
54
|
+
|
|
55
|
+
def _action(self, endpoint_id: str, action: str):
|
|
56
|
+
return self.transport.request("POST", f"/v1/endpoints/{_id(endpoint_id)}/{action}", {})
|
|
57
|
+
|
|
58
|
+
def wake(self, endpoint_id: str):
|
|
59
|
+
return self._action(endpoint_id, "wake")
|
|
60
|
+
|
|
61
|
+
def pause(self, endpoint_id: str):
|
|
62
|
+
return self._action(endpoint_id, "pause")
|
|
63
|
+
|
|
64
|
+
def resume(self, endpoint_id: str):
|
|
65
|
+
return self._action(endpoint_id, "resume")
|
|
66
|
+
|
|
67
|
+
def delete(self, endpoint_id: str):
|
|
68
|
+
return self.transport.request("DELETE", f"/v1/endpoints/{_id(endpoint_id)}")
|
|
69
|
+
|
|
70
|
+
def wait(self, endpoint_id: str, *, timeout: float = 1800, poll_interval: float = 5):
|
|
71
|
+
_wait_options(timeout, poll_interval)
|
|
72
|
+
deadline = time.monotonic() + timeout
|
|
73
|
+
while True:
|
|
74
|
+
result = self.get(endpoint_id)
|
|
75
|
+
if result["status"] == "READY":
|
|
76
|
+
return result
|
|
77
|
+
if result["status"] in {"FAILED", "DELETED", "DELETING", "PAUSED", "SUSPENDED"}:
|
|
78
|
+
raise RemoteError("ENDPOINT_NOT_READY")
|
|
79
|
+
remaining = deadline - time.monotonic()
|
|
80
|
+
if remaining <= 0:
|
|
81
|
+
raise TimeoutError(f"Endpoint {endpoint_id} did not become ready")
|
|
82
|
+
time.sleep(min(remaining, poll_interval))
|
|
83
|
+
|
|
84
|
+
def generate(self, model: str, prompt: str, *, job_id: str | None = None, **options: Any):
|
|
85
|
+
"""Return the API completion; transient cold-start errors remain observable."""
|
|
86
|
+
if job_id is None:
|
|
87
|
+
raise ValueError('job_id is required; use the Job ID returned by endpoint creation')
|
|
88
|
+
return self.transport.request("POST", f"/v1/job/{_id(job_id)}/Inference/chat/completions", {
|
|
89
|
+
"model": model, "messages": [{"role": "user", "content": prompt}], **options,
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class TemporaryResources:
|
|
94
|
+
"""Register on the server before work; wait for cleanup on every context exit.
|
|
95
|
+
|
|
96
|
+
The bounded server TTL survives a killed/disconnected process. Only resource
|
|
97
|
+
creation through this scope is enrolled; passing an existing ID never enrolls
|
|
98
|
+
it. Keep ``scope_id`` to retry cleanup after a client timeout.
|
|
99
|
+
"""
|
|
100
|
+
cleanup_result: dict[str, Any] | None
|
|
101
|
+
|
|
102
|
+
def __init__(self, client: Any, *, ttl_seconds: int = 1800, request_id: str | None = None, cleanup_timeout: float = 600, poll_interval: float = 2):
|
|
103
|
+
if isinstance(ttl_seconds, bool) or not isinstance(ttl_seconds, int) or not 60 <= ttl_seconds <= 86400:
|
|
104
|
+
raise ValueError("ttl_seconds must be an integer between 60 and 86400")
|
|
105
|
+
_wait_options(cleanup_timeout, poll_interval)
|
|
106
|
+
self.client, self.ttl_seconds = client, ttl_seconds
|
|
107
|
+
self.request_id = request_id or str(uuid4())
|
|
108
|
+
self.cleanup_timeout, self.poll_interval = cleanup_timeout, poll_interval
|
|
109
|
+
self.scope_id, self.cleanup_result = None, None
|
|
110
|
+
|
|
111
|
+
def __enter__(self):
|
|
112
|
+
if self.scope_id is not None:
|
|
113
|
+
raise RuntimeError("A temporary scope can only be entered once")
|
|
114
|
+
scope = self.client._transport.request("POST", self.client.operations.project_path + "/temporary-resource-scopes",
|
|
115
|
+
{"request_id": self.request_id, "ttl_seconds": self.ttl_seconds})
|
|
116
|
+
self.scope_id = scope["scope_id"]
|
|
117
|
+
self.endpoints = Endpoints(self.client._transport, self.scope_id)
|
|
118
|
+
self.adapters = Adapters(self.client._transport, self.scope_id)
|
|
119
|
+
return self
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def path(self):
|
|
123
|
+
if self.scope_id is None:
|
|
124
|
+
raise RuntimeError("Enter the temporary resource scope before using it")
|
|
125
|
+
return self.client.operations.project_path + "/temporary-resource-scopes/" + _id(self.scope_id)
|
|
126
|
+
|
|
127
|
+
def get(self):
|
|
128
|
+
return self.client._transport.request("GET", self.path)
|
|
129
|
+
|
|
130
|
+
def retain(self, resources: list[dict[str, str]]):
|
|
131
|
+
return self.client._transport.request("POST", self.path + "/retain", {"resources": resources})
|
|
132
|
+
|
|
133
|
+
def cleanup(self):
|
|
134
|
+
self.cleanup_result = self.client._transport.request("DELETE", self.path)
|
|
135
|
+
try:
|
|
136
|
+
self.cleanup_result = self.client.operations.wait(self.cleanup_result["operation_id"],
|
|
137
|
+
timeout=self.cleanup_timeout, poll_interval=self.poll_interval)
|
|
138
|
+
except CleanupError as error:
|
|
139
|
+
self.cleanup_result = error.operation
|
|
140
|
+
raise
|
|
141
|
+
return self.cleanup_result
|
|
142
|
+
|
|
143
|
+
def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None) -> bool:
|
|
144
|
+
try:
|
|
145
|
+
self.cleanup()
|
|
146
|
+
except BaseException as cleanup_error:
|
|
147
|
+
if exc is None:
|
|
148
|
+
raise
|
|
149
|
+
# Preserve both failures: the primary error is never silently replaced.
|
|
150
|
+
exc.add_note(f"Cleanup remains incomplete for scope {self.scope_id}; {type(cleanup_error).__name__}. Server TTL remains active.")
|
|
151
|
+
raise exc from cleanup_error
|
|
152
|
+
return False
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Literal highest-priority runtime parameters; no shell or environment expansion."""
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
from collections.abc import Iterable, Sequence
|
|
6
|
+
from copy import deepcopy
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
# Keep aligned with the public server runtime-patch boundary, not resource selectors.
|
|
10
|
+
PROTECTED = frozenset('PATH HOME SHELL ENV BASH_ENV PYTHONPATH PYTHONHOME NODE_OPTIONS EXECUTOR PROVIDER '
|
|
11
|
+
'RUNNER RUNNER_REF RUNNER_SOURCE REVISION SOURCE SOURCE_HASH SHA256 IMAGE DOCKER_IMAGE ENTRYPOINT COMMAND RUN_COMMAND '
|
|
12
|
+
'PROFILE PROFILE_ID PROFILE_NAME PROFILE_VERSION PROFILE_SHA256 PROFILE_REF DATASET DATASET_REF TASK TASK_REF '
|
|
13
|
+
'ACL TENANT TENANT_ID WORKSPACE_ID PROJECT_ID USER_ID BUDGET LIMITS QUOTA EXECUTION ENVIRONMENT'.split())
|
|
14
|
+
PREFIXES = ('ENVLOOP_', 'ENVPLATFORM_', 'JOB_MASTER_', 'LD_', 'DYLD_')
|
|
15
|
+
UNSAFE = {'__proto__', 'constructor', 'prototype'}
|
|
16
|
+
MESSAGE = 'expected a runtime parameter JSON object or KEY=VALUE; invalid or protected parameter'
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _check_json(value: Any, depth: int = 0) -> None:
|
|
20
|
+
if depth > 32 or isinstance(value, str) and '\x00' in value:
|
|
21
|
+
raise ValueError(MESSAGE)
|
|
22
|
+
if isinstance(value, dict):
|
|
23
|
+
for key, item in value.items():
|
|
24
|
+
if not isinstance(key, str) or key in UNSAFE:
|
|
25
|
+
raise ValueError(MESSAGE)
|
|
26
|
+
_check_json(item, depth + 1)
|
|
27
|
+
elif isinstance(value, list):
|
|
28
|
+
for item in value:
|
|
29
|
+
_check_json(item, depth + 1)
|
|
30
|
+
elif value is not None and type(value) not in (str, int, float, bool):
|
|
31
|
+
raise ValueError(MESSAGE)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def validate_params(value: Any) -> dict[str, Any]:
|
|
35
|
+
try:
|
|
36
|
+
if not isinstance(value, dict):
|
|
37
|
+
raise ValueError
|
|
38
|
+
_check_json(value)
|
|
39
|
+
raw = json.dumps(value, ensure_ascii=False, allow_nan=False)
|
|
40
|
+
if len(raw.encode()) > 65536 or '\x00' in raw or any('\x00' in k for k in value):
|
|
41
|
+
raise ValueError
|
|
42
|
+
result: dict[str, Any] = {}
|
|
43
|
+
for key, item in value.items():
|
|
44
|
+
target = key.upper()
|
|
45
|
+
if (not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', key, re.ASCII)
|
|
46
|
+
or target in PROTECTED or target.startswith(PREFIXES) or target in result):
|
|
47
|
+
raise ValueError
|
|
48
|
+
result[target] = deepcopy(item)
|
|
49
|
+
return result
|
|
50
|
+
except (ValueError, TypeError, RecursionError, OverflowError):
|
|
51
|
+
raise ValueError(MESSAGE) from None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def merge_params(base: Any, explicit: Any) -> dict[str, Any]:
|
|
55
|
+
return validate_params(validate_params(base) | validate_params(explicit))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _pairs(pairs: Iterable[tuple[str, Any]]) -> dict[str, Any]:
|
|
59
|
+
result = {}
|
|
60
|
+
for key, value in pairs:
|
|
61
|
+
if key in result:
|
|
62
|
+
raise ValueError(MESSAGE)
|
|
63
|
+
result[key] = value
|
|
64
|
+
return result
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def parse_params(text: str) -> dict[str, Any]:
|
|
68
|
+
try:
|
|
69
|
+
if not text.lstrip().startswith(('{', '[', '"')) and '=' in text:
|
|
70
|
+
key, value = text.split('=', 1)
|
|
71
|
+
return validate_params({key: value})
|
|
72
|
+
return validate_params(json.loads(text, object_pairs_hook=_pairs))
|
|
73
|
+
except (ValueError, TypeError, RecursionError):
|
|
74
|
+
raise argparse.ArgumentTypeError(MESSAGE) from None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class ParamsAction(argparse.Action):
|
|
78
|
+
def __call__(self, parser: argparse.ArgumentParser, namespace: argparse.Namespace,
|
|
79
|
+
values: str | Sequence[Any] | None, option_string: str | None = None) -> None:
|
|
80
|
+
try:
|
|
81
|
+
setattr(namespace, self.dest, merge_params(getattr(namespace, self.dest, None) or {}, values))
|
|
82
|
+
except ValueError:
|
|
83
|
+
raise argparse.ArgumentError(self, MESSAGE) from None
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Validate Sandbox Runner selections without resolving platform assets."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def validate_image(value: Any) -> str:
|
|
8
|
+
if (not isinstance(value, str) or len(value) > 512 or not re.fullmatch(
|
|
9
|
+
r'(?:[a-z0-9]+(?:(?:[._]|__|[-]+)[a-z0-9]+)*(?::[0-9]+)?/)*'
|
|
10
|
+
r'[a-z0-9]+(?:(?:[._]|__|[-]+)[a-z0-9]+)*'
|
|
11
|
+
r'(?::[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?(?:@sha256:[a-fA-F0-9]{64})?', value)):
|
|
12
|
+
raise ValueError('Sandbox image must be a Docker image name, optionally with a tag or sha256 digest')
|
|
13
|
+
return value
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def sandbox_runner_selector(value: Any) -> str:
|
|
17
|
+
if not isinstance(value, str) or not value or len(value) > 1024:
|
|
18
|
+
raise ValueError('Sandbox requires --runner NAME[@REVISION]')
|
|
19
|
+
prefix = 'envloop://runners/'
|
|
20
|
+
selector = value[len(prefix):] if value.startswith(prefix) else value
|
|
21
|
+
name, at, revision = selector.partition('@')
|
|
22
|
+
if not re.fullmatch(r'[a-z0-9][a-z0-9._-]*(?:/[a-z0-9][a-z0-9._-]*)?', name):
|
|
23
|
+
raise ValueError('Sandbox runner must be a short name or namespace/name[@REVISION]')
|
|
24
|
+
if at:
|
|
25
|
+
from envloop_experiment.client import runner_selector
|
|
26
|
+
|
|
27
|
+
runner_selector(f'{name if "/" in name else "envloop/" + name}@{revision}')
|
|
28
|
+
return selector
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Thin Train Job transport using the platform's unified Job contract."""
|
|
2
|
+
import math
|
|
3
|
+
import time
|
|
4
|
+
from typing import Any
|
|
5
|
+
from urllib.parse import quote, urlencode
|
|
6
|
+
|
|
7
|
+
from envloop.client import INTERACTIVE_TRAINING_REMOVED as INTERACTIVE_TRAINING_REMOVED
|
|
8
|
+
from envloop.http import RemoteError
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TrainJobs:
|
|
12
|
+
def __init__(self, transport: Any):
|
|
13
|
+
self.transport = transport
|
|
14
|
+
from .http import configure_transport
|
|
15
|
+
configure_transport(self.transport)
|
|
16
|
+
|
|
17
|
+
def models(self) -> dict[str, Any]:
|
|
18
|
+
"""Discover configured managed recipes; admission and GPU readiness remain separate."""
|
|
19
|
+
try:
|
|
20
|
+
return self.transport.request('GET', '/v1/job/training-models')
|
|
21
|
+
except RemoteError as error:
|
|
22
|
+
if error.status_code == 404:
|
|
23
|
+
raise RuntimeError(
|
|
24
|
+
"This platform does not serve /v1/job/training-models; upgrade the platform to a "
|
|
25
|
+
"version that matches this client (managed training discovery is unavailable).") from error
|
|
26
|
+
raise
|
|
27
|
+
|
|
28
|
+
@staticmethod
|
|
29
|
+
def _view(result: dict[str, Any]):
|
|
30
|
+
return {**(result.get('training') or {}), **result}
|
|
31
|
+
|
|
32
|
+
def submit(self, request: dict[str, Any]) -> dict[str, Any]:
|
|
33
|
+
"""Submit one managed spec or ``{request_id, trials: [{name?, spec}, ...]}``.
|
|
34
|
+
|
|
35
|
+
Each Trial spec is an independent managed recipe or published Runner
|
|
36
|
+
(``{runner: "namespace/name@REVISION", resources?: {...}, ...}``).
|
|
37
|
+
Optional ``tags`` is a bounded string map for either recipe or Runner specs.
|
|
38
|
+
Runner ``command`` is the declared run_command parameter, never an
|
|
39
|
+
entrypoint override. Validation, defaults and execution selection belong
|
|
40
|
+
to the platform. The training view retains the server's complete response.
|
|
41
|
+
"""
|
|
42
|
+
if not isinstance(request, dict) or request.get("type", "train") != "train":
|
|
43
|
+
raise ValueError("Train Job request must be a JSON object")
|
|
44
|
+
return self._view(self.transport.request("POST", "/v1/job", request | {"type": "train"}))
|
|
45
|
+
|
|
46
|
+
def status(self, job_id: str) -> dict[str, Any]:
|
|
47
|
+
return self._view(self.transport.request("GET", f"/v1/job/{quote(job_id, safe='')}") )
|
|
48
|
+
|
|
49
|
+
def result(self, job_id: str) -> dict[str, Any]:
|
|
50
|
+
return self._view(self.transport.request('GET', f'/v1/job/{quote(job_id, safe="")}/result'))
|
|
51
|
+
|
|
52
|
+
def logs(self, job_id: str, *, source: str = "all", cursor: str | None = None, tail: int | None = None, limit: int = 200, process: str | None = None) -> dict[str, Any]:
|
|
53
|
+
"""Read a log page; cursors retain the server's source and capture metadata.
|
|
54
|
+
|
|
55
|
+
``process="trainer"`` reads only the trainer's step metric lines. Live step, loss and ETA
|
|
56
|
+
are ``status(job_id)["progress"]``.
|
|
57
|
+
"""
|
|
58
|
+
from envloop_experiment.job_logs import log_query
|
|
59
|
+
|
|
60
|
+
query = urlencode(log_query(source=source, cursor=cursor, tail=tail, limit=limit, process=process))
|
|
61
|
+
return self.transport.request("GET", f"/v1/job/{quote(job_id, safe='')}/logs?{query}")
|
|
62
|
+
|
|
63
|
+
def stop(self, job_id: str) -> dict[str, Any]:
|
|
64
|
+
return self._view(self.transport.request('POST', f'/v1/job/{quote(job_id, safe="")}/stop'))
|
|
65
|
+
|
|
66
|
+
def list(self, *, status: str | None = None, cursor: str | None = None, limit: int = 25) -> dict[str, Any]:
|
|
67
|
+
query = urlencode({key: value for key, value in {
|
|
68
|
+
"type": "train", "status": status, "cursor": cursor, "limit": limit,
|
|
69
|
+
}.items() if value is not None})
|
|
70
|
+
page = self.transport.request("GET", "/v1/job?" + query)
|
|
71
|
+
return {**page, "items": [self._view(item) for item in page["items"]]}
|
|
72
|
+
|
|
73
|
+
def checkpoints(self, *, train_job_id: str | None = None, job_id: str | None = None, run_id: str | None = None, cursor: str | None = None, limit: int = 25) -> dict[str, Any]:
|
|
74
|
+
query = urlencode({key: value for key, value in {
|
|
75
|
+
"train_job_id": train_job_id, "job_id": job_id, "run_id": run_id,
|
|
76
|
+
"cursor": cursor, "limit": limit,
|
|
77
|
+
}.items() if value is not None})
|
|
78
|
+
return self.transport.request("GET", "/v1/job/checkpoints?" + query)
|
|
79
|
+
|
|
80
|
+
def delete_checkpoint(self, checkpoint_id: str) -> dict[str, Any]:
|
|
81
|
+
return self.transport.request("DELETE", f"/v1/job/checkpoints/{quote(checkpoint_id, safe='')}")
|
|
82
|
+
|
|
83
|
+
def wait(self, job_id: str, *, timeout: float = 600) -> dict[str, Any]:
|
|
84
|
+
if not math.isfinite(timeout) or timeout < 0:
|
|
85
|
+
raise ValueError("timeout must be finite and non-negative")
|
|
86
|
+
deadline = time.monotonic() + timeout
|
|
87
|
+
while True:
|
|
88
|
+
result = self.status(job_id)
|
|
89
|
+
if result["status"] in {"SUCCEEDED", "FAILED", "CANCELLED", "completed", "succeeded", "failed", "cancelled"}:
|
|
90
|
+
return result
|
|
91
|
+
if time.monotonic() >= deadline:
|
|
92
|
+
raise TimeoutError("Train Job did not finish before timeout")
|
|
93
|
+
time.sleep(min(0.1, max(0, deadline - time.monotonic())))
|