risorius-rl 0.15.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- risorius_rl/__init__.py +22 -0
- risorius_rl/cli.py +272 -0
- risorius_rl/execution/__init__.py +1 -0
- risorius_rl/execution/attached.py +207 -0
- risorius_rl/execution/local.py +119 -0
- risorius_rl/execution/ray.py +117 -0
- risorius_rl/execution/run.py +473 -0
- risorius_rl/execution/runner.py +256 -0
- risorius_rl/execution/workers.py +85 -0
- risorius_rl/execution/workspace.py +6 -0
- risorius_rl/integrations/__init__.py +1 -0
- risorius_rl/integrations/pytorch.py +44 -0
- risorius_rl/observability/__init__.py +1 -0
- risorius_rl/observability/codec.py +119 -0
- risorius_rl/observability/config.py +118 -0
- risorius_rl/observability/passive_env.py +197 -0
- risorius_rl/observability/recording.py +458 -0
- risorius_rl/observability/session.py +538 -0
- risorius_rl/observability/spool.py +67 -0
- risorius_rl/observability/telemetry.py +222 -0
- risorius_rl/observability/transport.py +375 -0
- risorius_rl/py.typed +0 -0
- risorius_rl/pytorch.py +3 -0
- risorius_rl/runner.py +6 -0
- risorius_rl/runtime.py +3 -0
- risorius_rl/storage/__init__.py +1 -0
- risorius_rl/storage/datasets.py +84 -0
- risorius_rl/storage/evidence.py +217 -0
- risorius_rl/storage/files.py +183 -0
- risorius_rl/user_cli/__init__.py +1 -0
- risorius_rl/user_cli/auth.py +260 -0
- risorius_rl/user_cli/main.py +168 -0
- risorius_rl/user_cli/records.py +130 -0
- risorius_rl-0.15.1.dist-info/METADATA +78 -0
- risorius_rl-0.15.1.dist-info/RECORD +39 -0
- risorius_rl-0.15.1.dist-info/WHEEL +5 -0
- risorius_rl-0.15.1.dist-info/entry_points.txt +3 -0
- risorius_rl-0.15.1.dist-info/licenses/SDK-LICENSE +21 -0
- risorius_rl-0.15.1.dist-info/top_level.txt +1 -0
risorius_rl/__init__.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Record results without exposing platform or AWS credentials to research code."""
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
if TYPE_CHECKING:
|
|
6
|
+
from risorius_rl.execution.run import Run as Run
|
|
7
|
+
from risorius_rl.observability.session import RecordingSession as RecordingSession
|
|
8
|
+
|
|
9
|
+
__all__ = ["Run", "RecordingSession"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def __getattr__(name):
|
|
13
|
+
# Recording-only imports must not load execution or cloud management clients.
|
|
14
|
+
if name == "Run":
|
|
15
|
+
from risorius_rl.execution.run import Run
|
|
16
|
+
|
|
17
|
+
return Run
|
|
18
|
+
if name == "RecordingSession":
|
|
19
|
+
from risorius_rl.observability.session import RecordingSession
|
|
20
|
+
|
|
21
|
+
return RecordingSession
|
|
22
|
+
raise AttributeError(name)
|
risorius_rl/cli.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
"""Project-scoped research submission using the current AWS profile (SigV4)."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import uuid
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from urllib.parse import urlparse
|
|
10
|
+
|
|
11
|
+
import boto3
|
|
12
|
+
import httpx
|
|
13
|
+
import yaml
|
|
14
|
+
from botocore.auth import SigV4Auth
|
|
15
|
+
from botocore.awsrequest import AWSRequest
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def request(method, path, body=None, *, endpoint=None):
|
|
19
|
+
endpoint = (endpoint or os.environ.get("RL_API_URL", "")).rstrip("/")
|
|
20
|
+
parsed = urlparse(endpoint)
|
|
21
|
+
if (
|
|
22
|
+
parsed.scheme != "https"
|
|
23
|
+
or not re.fullmatch(
|
|
24
|
+
r"[a-z0-9]+\.execute-api\.ap-northeast-2\.amazonaws\.com", parsed.netloc
|
|
25
|
+
)
|
|
26
|
+
or parsed.path
|
|
27
|
+
):
|
|
28
|
+
raise ValueError("Set RL_API_URL to the platform's HTTPS API Gateway endpoint")
|
|
29
|
+
credentials = boto3.Session().get_credentials()
|
|
30
|
+
if credentials is None:
|
|
31
|
+
raise ValueError("Authenticate your AWS profile first")
|
|
32
|
+
content = json.dumps(body, allow_nan=False) if body is not None else ""
|
|
33
|
+
signed = AWSRequest(
|
|
34
|
+
method=method,
|
|
35
|
+
url=endpoint + path,
|
|
36
|
+
data=content,
|
|
37
|
+
headers={"Content-Type": "application/json"},
|
|
38
|
+
)
|
|
39
|
+
SigV4Auth(credentials.get_frozen_credentials(), "execute-api", "ap-northeast-2").add_auth(
|
|
40
|
+
signed
|
|
41
|
+
)
|
|
42
|
+
with httpx.Client(timeout=30, trust_env=False) as client:
|
|
43
|
+
response = client.request(
|
|
44
|
+
method, endpoint + path, content=content, headers=dict(signed.headers)
|
|
45
|
+
)
|
|
46
|
+
if response.is_error:
|
|
47
|
+
raise RuntimeError(f"Platform returned {response.status_code}: {response.text}")
|
|
48
|
+
return response.json()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def main():
|
|
52
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
53
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
54
|
+
local = commands.add_parser("local")
|
|
55
|
+
local.add_argument("spec", type=Path)
|
|
56
|
+
local.add_argument("--output", type=Path, default=Path("artifacts/local"))
|
|
57
|
+
local.add_argument(
|
|
58
|
+
"--gpu-metrics",
|
|
59
|
+
action="store_true",
|
|
60
|
+
help="Sample visible NVIDIA GPUs; does not allocate a GPU",
|
|
61
|
+
)
|
|
62
|
+
prepare_command = commands.add_parser(
|
|
63
|
+
"workspace-prepare", help="Prepare a private run session and an owner approval request"
|
|
64
|
+
)
|
|
65
|
+
prepare_command.add_argument("spec", type=Path)
|
|
66
|
+
prepare_command.add_argument("--workspace", required=True)
|
|
67
|
+
prepare_command.add_argument("--session", type=Path, required=True)
|
|
68
|
+
workspace = commands.add_parser(
|
|
69
|
+
"workspace-run", help="Record training in an existing GPU workspace"
|
|
70
|
+
)
|
|
71
|
+
workspace.add_argument("session", type=Path)
|
|
72
|
+
workspace.add_argument("--url", required=True)
|
|
73
|
+
workspace.add_argument("--ca-file", type=Path, required=True)
|
|
74
|
+
external_prepare = commands.add_parser(
|
|
75
|
+
"external-prepare", help="Prepare recording on an external GPU"
|
|
76
|
+
)
|
|
77
|
+
external_prepare.add_argument("spec", type=Path)
|
|
78
|
+
external_prepare.add_argument("--provider", required=True)
|
|
79
|
+
external_prepare.add_argument("--duration-minutes", type=int, default=60)
|
|
80
|
+
external_prepare.add_argument("--session", type=Path, required=True)
|
|
81
|
+
external_run = commands.add_parser(
|
|
82
|
+
"external-run", help="Run with an approved external recording session"
|
|
83
|
+
)
|
|
84
|
+
external_run.add_argument("session", type=Path)
|
|
85
|
+
external_run.add_argument("--url", required=True)
|
|
86
|
+
external_run.add_argument("--ca-file", type=Path)
|
|
87
|
+
submit = commands.add_parser("submit")
|
|
88
|
+
submit.add_argument("spec", type=Path)
|
|
89
|
+
submit.add_argument("--image", required=True)
|
|
90
|
+
submit.add_argument("--run-id", default=None)
|
|
91
|
+
register = commands.add_parser("register-image")
|
|
92
|
+
register.add_argument("--project", required=True)
|
|
93
|
+
register.add_argument("--image", required=True)
|
|
94
|
+
register.add_argument("--commit", required=True)
|
|
95
|
+
register.add_argument("--backend", choices=("job", "ray"), default="job")
|
|
96
|
+
build = commands.add_parser("build", help="Package an approved GitHub source commit")
|
|
97
|
+
build.add_argument("--project", required=True)
|
|
98
|
+
build.add_argument("--commit", required=True)
|
|
99
|
+
build.add_argument("--request-id", default=None)
|
|
100
|
+
build.add_argument("--dependencies", choices=("project", "cpu", "ray"), default="project")
|
|
101
|
+
builds = commands.add_parser("builds", help="List the latest source builds")
|
|
102
|
+
builds.add_argument("--project", required=True)
|
|
103
|
+
for name in ("build-get", "build-cancel"):
|
|
104
|
+
command = commands.add_parser(name)
|
|
105
|
+
command.add_argument("--project", required=True)
|
|
106
|
+
command.add_argument("build_id")
|
|
107
|
+
for name in ("get", "cancel", "result", "list", "recordings"):
|
|
108
|
+
command = commands.add_parser(name)
|
|
109
|
+
command.add_argument("--project", required=True)
|
|
110
|
+
if name != "list":
|
|
111
|
+
command.add_argument("run_id")
|
|
112
|
+
artifact = commands.add_parser("download")
|
|
113
|
+
artifact.add_argument("--project", required=True)
|
|
114
|
+
artifact.add_argument("run_id")
|
|
115
|
+
artifact.add_argument("name")
|
|
116
|
+
artifact.add_argument("--output", type=Path, required=True)
|
|
117
|
+
recording = commands.add_parser(
|
|
118
|
+
"download-recording", help="Download committed chunks with SHA-256 verification"
|
|
119
|
+
)
|
|
120
|
+
recording.add_argument("--project", required=True)
|
|
121
|
+
recording.add_argument("run_id")
|
|
122
|
+
recording.add_argument("--stream", required=True)
|
|
123
|
+
recording.add_argument("--output", type=Path, required=True)
|
|
124
|
+
bundle = commands.add_parser(
|
|
125
|
+
"download-run", help="Download all recording streams, summaries and runtime logs"
|
|
126
|
+
)
|
|
127
|
+
bundle.add_argument("--project", required=True)
|
|
128
|
+
bundle.add_argument("run_id")
|
|
129
|
+
bundle.add_argument("--output", type=Path, required=True)
|
|
130
|
+
args = parser.parse_args()
|
|
131
|
+
try:
|
|
132
|
+
if args.command == "external-prepare":
|
|
133
|
+
from risorius_rl.execution.attached import prepare
|
|
134
|
+
|
|
135
|
+
result = prepare(
|
|
136
|
+
yaml.safe_load(args.spec.read_text()),
|
|
137
|
+
None,
|
|
138
|
+
args.session,
|
|
139
|
+
provider=args.provider,
|
|
140
|
+
duration_minutes=args.duration_minutes,
|
|
141
|
+
)
|
|
142
|
+
print(json.dumps(result, indent=2))
|
|
143
|
+
return
|
|
144
|
+
if args.command == "external-run":
|
|
145
|
+
from risorius_rl.execution.attached import execute_attached
|
|
146
|
+
|
|
147
|
+
result, code = execute_attached(
|
|
148
|
+
args.session, args.url, args.ca_file, backend="external"
|
|
149
|
+
)
|
|
150
|
+
print(json.dumps(result, indent=2))
|
|
151
|
+
if code:
|
|
152
|
+
parser.exit(code, "External execution did not succeed\n")
|
|
153
|
+
return
|
|
154
|
+
if args.command == "workspace-prepare":
|
|
155
|
+
from risorius_rl.execution.workspace import prepare
|
|
156
|
+
|
|
157
|
+
result = prepare(yaml.safe_load(args.spec.read_text()), args.workspace, args.session)
|
|
158
|
+
print(json.dumps(result, indent=2))
|
|
159
|
+
return
|
|
160
|
+
if args.command == "workspace-run":
|
|
161
|
+
from risorius_rl.execution.workspace import execute_workspace
|
|
162
|
+
|
|
163
|
+
result, code = execute_workspace(args.session, args.url, args.ca_file)
|
|
164
|
+
print(json.dumps(result, indent=2))
|
|
165
|
+
if code:
|
|
166
|
+
parser.exit(code, "Workspace execution did not succeed\n")
|
|
167
|
+
return
|
|
168
|
+
if args.command in {"local", "submit"}:
|
|
169
|
+
spec = yaml.safe_load(args.spec.read_text())
|
|
170
|
+
if args.command == "local":
|
|
171
|
+
from risorius_rl.execution.local import execute_local
|
|
172
|
+
|
|
173
|
+
local_result, code = execute_local(spec, args.output, gpu_metrics=args.gpu_metrics)
|
|
174
|
+
if code:
|
|
175
|
+
parser.exit(code, f"Local execution failed; evidence: {local_result.parent}\n")
|
|
176
|
+
print(str(local_result))
|
|
177
|
+
return
|
|
178
|
+
run_id = args.run_id or uuid.uuid4().hex
|
|
179
|
+
print(f"Run ID: {run_id} (reuse --run-id if the response is lost)", flush=True)
|
|
180
|
+
result = request(
|
|
181
|
+
"POST",
|
|
182
|
+
f"/v1/projects/{spec['project']}/runs",
|
|
183
|
+
{"run_id": run_id, "image": args.image, "spec": spec},
|
|
184
|
+
)
|
|
185
|
+
else:
|
|
186
|
+
base = f"/v1/projects/{args.project}"
|
|
187
|
+
if args.command == "build":
|
|
188
|
+
request_id = args.request_id or uuid.uuid4().hex
|
|
189
|
+
print(
|
|
190
|
+
f"Build request ID: {request_id} (reuse --request-id within 5 minutes if the response is lost)",
|
|
191
|
+
flush=True,
|
|
192
|
+
)
|
|
193
|
+
result = request(
|
|
194
|
+
"POST",
|
|
195
|
+
base + "/builds",
|
|
196
|
+
{
|
|
197
|
+
"commit": args.commit,
|
|
198
|
+
"request_id": request_id,
|
|
199
|
+
"dependencies": args.dependencies,
|
|
200
|
+
},
|
|
201
|
+
)
|
|
202
|
+
elif args.command == "builds":
|
|
203
|
+
result = request("GET", base + "/builds")
|
|
204
|
+
elif args.command in {"build-get", "build-cancel"}:
|
|
205
|
+
path = base + "/builds/" + args.build_id
|
|
206
|
+
result = (
|
|
207
|
+
request("POST", path + "/cancel", {})
|
|
208
|
+
if args.command == "build-cancel"
|
|
209
|
+
else request("GET", path)
|
|
210
|
+
)
|
|
211
|
+
elif args.command == "register-image":
|
|
212
|
+
result = request(
|
|
213
|
+
"POST",
|
|
214
|
+
base + "/images",
|
|
215
|
+
{
|
|
216
|
+
"image": args.image,
|
|
217
|
+
"commit": args.commit,
|
|
218
|
+
**({"backend": "ray"} if args.backend == "ray" else {}),
|
|
219
|
+
},
|
|
220
|
+
)
|
|
221
|
+
elif args.command == "list":
|
|
222
|
+
result = request("GET", base + "/runs")
|
|
223
|
+
elif args.command == "download":
|
|
224
|
+
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", args.name):
|
|
225
|
+
raise ValueError("Invalid artifact name")
|
|
226
|
+
result = request("GET", base + "/runs/" + args.run_id + "/artifacts/" + args.name)
|
|
227
|
+
with httpx.Client(timeout=60, trust_env=False) as client:
|
|
228
|
+
response = client.get(result["url"])
|
|
229
|
+
response.raise_for_status()
|
|
230
|
+
with args.output.open("xb") as target:
|
|
231
|
+
target.write(response.content)
|
|
232
|
+
result = {"saved": str(args.output)}
|
|
233
|
+
elif args.command == "download-run":
|
|
234
|
+
from risorius_rl.storage.evidence import download_run
|
|
235
|
+
|
|
236
|
+
result = download_run(request, base + "/runs/" + args.run_id, args.output)
|
|
237
|
+
elif args.command == "download-recording":
|
|
238
|
+
prefix = base + "/runs/" + args.run_id
|
|
239
|
+
snapshot = request("GET", prefix + "/recordings")
|
|
240
|
+
stream = next(
|
|
241
|
+
(s for s in snapshot["streams"] if s["stream_id"] == args.stream), None
|
|
242
|
+
)
|
|
243
|
+
if stream is None or "manifest" not in stream:
|
|
244
|
+
raise ValueError(
|
|
245
|
+
"Recording stream is unavailable or registration is incomplete"
|
|
246
|
+
)
|
|
247
|
+
from risorius_rl.storage.evidence import download_stream
|
|
248
|
+
|
|
249
|
+
with httpx.Client(timeout=60, trust_env=False) as client:
|
|
250
|
+
download_stream(request, client, prefix, stream, args.output)
|
|
251
|
+
(args.output / "snapshot.json").write_text(json.dumps(snapshot, indent=2))
|
|
252
|
+
result = {
|
|
253
|
+
"saved": str(args.output),
|
|
254
|
+
"chunks": stream["next_chunk"],
|
|
255
|
+
"closed": stream["state"]["closed"],
|
|
256
|
+
}
|
|
257
|
+
else:
|
|
258
|
+
path = base + "/runs/" + args.run_id
|
|
259
|
+
if args.command in {"cancel", "result", "recordings"}:
|
|
260
|
+
path += "/" + args.command
|
|
261
|
+
result = request(
|
|
262
|
+
"POST" if args.command == "cancel" else "GET",
|
|
263
|
+
path,
|
|
264
|
+
{} if args.command == "cancel" else None,
|
|
265
|
+
)
|
|
266
|
+
print(json.dumps(result, indent=2, ensure_ascii=False))
|
|
267
|
+
except (ValueError, RuntimeError, httpx.HTTPError) as error:
|
|
268
|
+
parser.exit(1, str(error) + "\n")
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
if __name__ == "__main__":
|
|
272
|
+
main()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Run lifecycle and managed subprocess/Ray execution."""
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Prepare a scoped approval and run research on an already available machine."""
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import secrets
|
|
9
|
+
import signal
|
|
10
|
+
import stat
|
|
11
|
+
import subprocess
|
|
12
|
+
import sys
|
|
13
|
+
import time
|
|
14
|
+
import uuid
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from urllib.parse import urlparse
|
|
17
|
+
|
|
18
|
+
import httpx
|
|
19
|
+
|
|
20
|
+
from risorius_rl.execution.local import source_identity
|
|
21
|
+
from risorius_rl.observability.codec import json_bytes
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def prepare(spec, workspace_id, session_file, *, provider=None, duration_minutes=60):
|
|
25
|
+
backend = "external" if provider is not None else "workspace"
|
|
26
|
+
if backend == "workspace":
|
|
27
|
+
workspace_id = str(uuid.UUID(workspace_id))
|
|
28
|
+
elif (
|
|
29
|
+
not isinstance(provider, str)
|
|
30
|
+
or not re.fullmatch(r"[a-z][a-z0-9-]{0,31}", provider)
|
|
31
|
+
or type(duration_minutes) is not int
|
|
32
|
+
or not 1 <= duration_minutes <= 1440
|
|
33
|
+
):
|
|
34
|
+
raise ValueError("Use a provider name and a recording duration of 1..1440 minutes")
|
|
35
|
+
if not isinstance(spec, dict) or set(spec) - {
|
|
36
|
+
"project",
|
|
37
|
+
"kind",
|
|
38
|
+
"entrypoint",
|
|
39
|
+
"parameters",
|
|
40
|
+
"resources",
|
|
41
|
+
"backend",
|
|
42
|
+
}:
|
|
43
|
+
raise ValueError("Workspace runs support project, kind, entrypoint and parameters only")
|
|
44
|
+
if spec.get("backend", "job") != "job":
|
|
45
|
+
raise ValueError("Prepare a single-process job spec")
|
|
46
|
+
# Local validation is for usability; the Gateway independently validates the approved payload.
|
|
47
|
+
if not re.fullmatch(r"[a-z][a-z0-9-]{0,47}", spec.get("project", "")):
|
|
48
|
+
raise ValueError("Invalid project")
|
|
49
|
+
if spec.get("kind") not in {"rollout", "world-model"} or not isinstance(
|
|
50
|
+
spec.get("parameters"), dict
|
|
51
|
+
):
|
|
52
|
+
raise ValueError("Use rollout/world-model and a parameters dictionary")
|
|
53
|
+
argv = spec.get("entrypoint")
|
|
54
|
+
if (
|
|
55
|
+
not isinstance(argv, list)
|
|
56
|
+
or not argv
|
|
57
|
+
or not all(isinstance(arg, str) and arg and "\x00" not in arg for arg in argv)
|
|
58
|
+
):
|
|
59
|
+
raise ValueError("Use an entrypoint argument list")
|
|
60
|
+
spec = {**spec, "backend": backend, "resources": backend}
|
|
61
|
+
payload = json_bytes({"spec": spec, "source": source_identity()})
|
|
62
|
+
if len(payload) > 32 * 1024:
|
|
63
|
+
raise ValueError("Workspace payload exceeds 32 KiB")
|
|
64
|
+
token = secrets.token_urlsafe(32)
|
|
65
|
+
run_id = uuid.uuid4().hex
|
|
66
|
+
session = {"run_id": run_id, "token": token, "payload": payload.decode()}
|
|
67
|
+
fd = os.open(session_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
68
|
+
with os.fdopen(fd, "w") as output:
|
|
69
|
+
json.dump(session, output)
|
|
70
|
+
return {
|
|
71
|
+
"run_id": run_id,
|
|
72
|
+
"session_file": str(Path(session_file).resolve()),
|
|
73
|
+
"approval": {
|
|
74
|
+
"method": "PUT",
|
|
75
|
+
"path": (
|
|
76
|
+
f"/v1/projects/{spec['project']}/workspaces/{workspace_id}/runs/{run_id}/grant"
|
|
77
|
+
if backend == "workspace"
|
|
78
|
+
else f"/v1/projects/{spec['project']}/external-runs/{run_id}/grant"
|
|
79
|
+
),
|
|
80
|
+
"body": {
|
|
81
|
+
"tokenSha256": hashlib.sha256(token.encode()).hexdigest(),
|
|
82
|
+
"payloadSha256": hashlib.sha256(payload).hexdigest(),
|
|
83
|
+
**(
|
|
84
|
+
{"provider": provider, "durationMinutes": duration_minutes}
|
|
85
|
+
if backend == "external"
|
|
86
|
+
else {}
|
|
87
|
+
),
|
|
88
|
+
},
|
|
89
|
+
},
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def execute_attached(session_file, url, ca_file, *, backend="workspace"):
|
|
94
|
+
"""A private session file is a run capability, never a general AWS or workspace key."""
|
|
95
|
+
if backend not in {"workspace", "external"}:
|
|
96
|
+
raise ValueError("Invalid execution backend")
|
|
97
|
+
parsed = urlparse(url)
|
|
98
|
+
if (
|
|
99
|
+
parsed.scheme != "https"
|
|
100
|
+
or not parsed.hostname
|
|
101
|
+
or parsed.username
|
|
102
|
+
or parsed.password
|
|
103
|
+
or parsed.query
|
|
104
|
+
or parsed.fragment
|
|
105
|
+
or parsed.path
|
|
106
|
+
not in ({"", "/", "/recording", "/recording/"} if backend == "external" else {"", "/"})
|
|
107
|
+
):
|
|
108
|
+
raise ValueError("Use the approved HTTPS recording Gateway origin")
|
|
109
|
+
url = url.rstrip("/")
|
|
110
|
+
fd = os.open(session_file, os.O_RDONLY | os.O_NOFOLLOW)
|
|
111
|
+
with os.fdopen(fd) as source:
|
|
112
|
+
meta = os.fstat(source.fileno())
|
|
113
|
+
if (
|
|
114
|
+
not stat.S_ISREG(meta.st_mode)
|
|
115
|
+
or meta.st_uid != os.getuid()
|
|
116
|
+
or meta.st_mode & 0o077
|
|
117
|
+
or meta.st_size > 64 * 1024
|
|
118
|
+
):
|
|
119
|
+
raise ValueError("Session file must be owned by you with mode 0600 and <=64 KiB")
|
|
120
|
+
session = json.load(source)
|
|
121
|
+
run_id, token, payload = session["run_id"], session["token"], session["payload"]
|
|
122
|
+
if not re.fullmatch(r"[0-9a-f]{32}", run_id) or not re.fullmatch(r"[A-Za-z0-9_-]{43}", token):
|
|
123
|
+
raise ValueError("Invalid workspace session")
|
|
124
|
+
request_payload = json.loads(payload)
|
|
125
|
+
if request_payload["spec"].get("backend") != backend:
|
|
126
|
+
raise ValueError("Session belongs to a different execution backend")
|
|
127
|
+
with httpx.Client(
|
|
128
|
+
verify=str(ca_file.resolve()) if ca_file else True,
|
|
129
|
+
timeout=15,
|
|
130
|
+
trust_env=False,
|
|
131
|
+
headers={"Authorization": "Bearer " + token},
|
|
132
|
+
follow_redirects=False,
|
|
133
|
+
) as client:
|
|
134
|
+
|
|
135
|
+
def send(path, content, *, retry=True):
|
|
136
|
+
for attempt in range(3):
|
|
137
|
+
try:
|
|
138
|
+
response = client.post(url + path, content=content)
|
|
139
|
+
# The ingress emits this header only before forwarding the request.
|
|
140
|
+
rejected = (
|
|
141
|
+
backend == "external"
|
|
142
|
+
and response.status_code == 429
|
|
143
|
+
and response.headers.get("Retry-After") == "5"
|
|
144
|
+
)
|
|
145
|
+
if rejected and attempt < 2:
|
|
146
|
+
time.sleep(5)
|
|
147
|
+
continue
|
|
148
|
+
if response.status_code in {429, 500, 502, 503, 504} and retry and attempt < 2:
|
|
149
|
+
time.sleep(attempt + 1)
|
|
150
|
+
continue
|
|
151
|
+
# Do not include a response body or session contents in error logs.
|
|
152
|
+
if response.status_code != 200:
|
|
153
|
+
raise RuntimeError(
|
|
154
|
+
f"Workspace Gateway returned HTTP {response.status_code}"
|
|
155
|
+
)
|
|
156
|
+
return response.json()
|
|
157
|
+
except httpx.TransportError:
|
|
158
|
+
if not retry or attempt == 2:
|
|
159
|
+
raise RuntimeError(
|
|
160
|
+
"Workspace Gateway transport failed; execution state may be unknown"
|
|
161
|
+
) from None
|
|
162
|
+
time.sleep(attempt + 1)
|
|
163
|
+
raise RuntimeError("Workspace Gateway did not acknowledge request")
|
|
164
|
+
|
|
165
|
+
execution = send(f"/{backend}-runs/{run_id}", payload.encode())
|
|
166
|
+
if (
|
|
167
|
+
execution["spec"] != request_payload["spec"]
|
|
168
|
+
or execution["source"] != request_payload["source"]
|
|
169
|
+
):
|
|
170
|
+
raise RuntimeError("Gateway execution differs from prepared payload")
|
|
171
|
+
# Only a definite ingress rejection is safe to retry for a one-time start.
|
|
172
|
+
send(f"/worker/{run_id}/{backend}-start", b"{}", retry=False)
|
|
173
|
+
env = {key: value for key, value in os.environ.items() if not key.startswith("RL_")}
|
|
174
|
+
env.update(
|
|
175
|
+
RL_RUN_ID=run_id,
|
|
176
|
+
RL_PROJECT_ID=execution["spec"]["project"],
|
|
177
|
+
RL_RUN_TOKEN=token,
|
|
178
|
+
RL_WORKER_URL=url,
|
|
179
|
+
RL_RECORDING_ENABLED="1",
|
|
180
|
+
RL_EXECUTION_B64=base64.b64encode(json_bytes(execution["spec"])).decode(),
|
|
181
|
+
PATH=str(Path(sys.executable).parent) + os.pathsep + os.environ.get("PATH", ""),
|
|
182
|
+
)
|
|
183
|
+
if ca_file:
|
|
184
|
+
env["RL_CA_FILE"] = str(ca_file.resolve())
|
|
185
|
+
if execution["image"]:
|
|
186
|
+
env["RL_IMAGE_DIGEST"] = execution["image"]
|
|
187
|
+
if execution["source"]["commit"]:
|
|
188
|
+
env["RL_SOURCE_COMMIT"] = execution["source"]["commit"]
|
|
189
|
+
print(f"{backend.capitalize()} run: {run_id}", file=sys.stderr, flush=True)
|
|
190
|
+
code = 1
|
|
191
|
+
try:
|
|
192
|
+
with subprocess.Popen(
|
|
193
|
+
[sys.executable, "-m", "risorius_rl.execution.runner"],
|
|
194
|
+
env=env,
|
|
195
|
+
start_new_session=True,
|
|
196
|
+
) as process:
|
|
197
|
+
try:
|
|
198
|
+
code = process.wait()
|
|
199
|
+
except KeyboardInterrupt:
|
|
200
|
+
process.send_signal(signal.SIGTERM)
|
|
201
|
+
process.wait()
|
|
202
|
+
code = 130
|
|
203
|
+
finally:
|
|
204
|
+
outcome = send(f"/worker/{run_id}/{backend}-finish", json_bytes({"exit_code": code}))
|
|
205
|
+
return {"run_id": run_id, "state": outcome["state"]}, 0 if code == 0 and outcome[
|
|
206
|
+
"state"
|
|
207
|
+
] == "succeeded" else 1
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Run research code on the current machine and retain local execution evidence.
|
|
2
|
+
|
|
3
|
+
This path neither allocates resources nor authenticates an execution to the server.
|
|
4
|
+
Source metadata is diagnostic and must never be treated as a trusted build identity.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import base64
|
|
8
|
+
import hashlib
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import signal
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
import time
|
|
15
|
+
import uuid
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def source_identity() -> dict:
|
|
20
|
+
"""Capture HEAD and dirty state without exporting URLs, paths or source contents."""
|
|
21
|
+
try:
|
|
22
|
+
commit = subprocess.run(
|
|
23
|
+
["git", "rev-parse", "--verify", "HEAD"],
|
|
24
|
+
capture_output=True,
|
|
25
|
+
text=True,
|
|
26
|
+
check=True,
|
|
27
|
+
timeout=5,
|
|
28
|
+
).stdout.strip()
|
|
29
|
+
status = subprocess.run(
|
|
30
|
+
["git", "status", "--porcelain", "--untracked-files=normal"],
|
|
31
|
+
capture_output=True,
|
|
32
|
+
check=True,
|
|
33
|
+
timeout=5,
|
|
34
|
+
).stdout
|
|
35
|
+
return {"commit": commit, "dirty": bool(status), "verified_by_platform": False}
|
|
36
|
+
except (OSError, subprocess.SubprocessError):
|
|
37
|
+
return {"commit": None, "dirty": None, "verified_by_platform": False}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def execute_local(spec: dict, output: Path, *, gpu_metrics: bool = False) -> tuple[Path, int]:
|
|
41
|
+
"""Keep process outcome separate from a result written before a subsequent crash."""
|
|
42
|
+
if not isinstance(spec, dict) or spec.get("backend", "job") != "job":
|
|
43
|
+
raise ValueError("Local execution requires a job spec; use submit for managed Ray")
|
|
44
|
+
if not isinstance(spec.get("project"), str) or not spec["project"]:
|
|
45
|
+
raise ValueError("Set a project in the execution spec")
|
|
46
|
+
if spec.get("kind") not in {"rollout", "world-model"}:
|
|
47
|
+
raise ValueError("Use kind rollout or world-model")
|
|
48
|
+
if not isinstance(spec.get("parameters"), dict):
|
|
49
|
+
raise ValueError("Set parameters to a dictionary in the execution spec")
|
|
50
|
+
if (
|
|
51
|
+
not isinstance(spec.get("entrypoint"), list)
|
|
52
|
+
or not spec["entrypoint"]
|
|
53
|
+
or any(not isinstance(arg, str) or not arg or "\x00" in arg for arg in spec["entrypoint"])
|
|
54
|
+
):
|
|
55
|
+
raise ValueError("Use a nonempty entrypoint argument list")
|
|
56
|
+
content = json.dumps(spec, allow_nan=False).encode()
|
|
57
|
+
source = source_identity()
|
|
58
|
+
run_id = uuid.uuid4().hex
|
|
59
|
+
directory = output.resolve() / run_id
|
|
60
|
+
directory.mkdir(parents=True, mode=0o700, exist_ok=False)
|
|
61
|
+
evidence = {
|
|
62
|
+
"schema_version": 1,
|
|
63
|
+
"mode": "local",
|
|
64
|
+
"run_id": run_id,
|
|
65
|
+
"project": spec["project"],
|
|
66
|
+
"source": source,
|
|
67
|
+
"spec_sha256": hashlib.sha256(content).hexdigest(),
|
|
68
|
+
"gpu_metrics_requested": gpu_metrics,
|
|
69
|
+
"started_at": time.time(),
|
|
70
|
+
"state": "running",
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
def save():
|
|
74
|
+
staging = directory / "execution.json.tmp"
|
|
75
|
+
staging.write_text(json.dumps(evidence, indent=2) + "\n")
|
|
76
|
+
staging.replace(directory / "execution.json")
|
|
77
|
+
|
|
78
|
+
save()
|
|
79
|
+
# A local run started inside a managed worker must not reuse its identity or token.
|
|
80
|
+
env = {key: value for key, value in os.environ.items() if not key.startswith("RL_")}
|
|
81
|
+
env.update(
|
|
82
|
+
RL_RUN_ID=run_id,
|
|
83
|
+
RL_PROJECT_ID=spec["project"],
|
|
84
|
+
RL_EXECUTION_B64=base64.b64encode(content).decode(),
|
|
85
|
+
RL_LOCAL_OUTPUT=str(directory),
|
|
86
|
+
RL_GPU_METRICS="1" if gpu_metrics else "0",
|
|
87
|
+
PATH=str(Path(sys.executable).parent) + os.pathsep + os.environ.get("PATH", ""),
|
|
88
|
+
)
|
|
89
|
+
if source["commit"]:
|
|
90
|
+
env["RL_SOURCE_COMMIT"] = source["commit"]
|
|
91
|
+
print(f"Local evidence: {directory}", file=sys.stderr, flush=True)
|
|
92
|
+
code = 1
|
|
93
|
+
try:
|
|
94
|
+
with subprocess.Popen(
|
|
95
|
+
[sys.executable, "-m", "risorius_rl.execution.runner"],
|
|
96
|
+
env=env,
|
|
97
|
+
start_new_session=True,
|
|
98
|
+
) as process:
|
|
99
|
+
try:
|
|
100
|
+
code = process.wait()
|
|
101
|
+
except KeyboardInterrupt:
|
|
102
|
+
# The runner forwards TERM to its research process group and flushes its logs.
|
|
103
|
+
process.send_signal(signal.SIGTERM)
|
|
104
|
+
process.wait()
|
|
105
|
+
code = 130
|
|
106
|
+
except OSError as error:
|
|
107
|
+
evidence["error_type"] = type(error).__name__
|
|
108
|
+
finally:
|
|
109
|
+
result = directory / "result.json"
|
|
110
|
+
if code == 0 and not result.is_file():
|
|
111
|
+
code = 1
|
|
112
|
+
evidence["error_type"] = "MissingResult"
|
|
113
|
+
evidence.update(
|
|
114
|
+
state="succeeded" if code == 0 else "failed",
|
|
115
|
+
exit_code=code,
|
|
116
|
+
finished_at=time.time(),
|
|
117
|
+
)
|
|
118
|
+
save()
|
|
119
|
+
return result, code if code >= 0 else 128 - code
|