risorius-rl 0.15.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. risorius_rl/__init__.py +22 -0
  2. risorius_rl/cli.py +272 -0
  3. risorius_rl/execution/__init__.py +1 -0
  4. risorius_rl/execution/attached.py +207 -0
  5. risorius_rl/execution/local.py +119 -0
  6. risorius_rl/execution/ray.py +117 -0
  7. risorius_rl/execution/run.py +473 -0
  8. risorius_rl/execution/runner.py +256 -0
  9. risorius_rl/execution/workers.py +85 -0
  10. risorius_rl/execution/workspace.py +6 -0
  11. risorius_rl/integrations/__init__.py +1 -0
  12. risorius_rl/integrations/pytorch.py +44 -0
  13. risorius_rl/observability/__init__.py +1 -0
  14. risorius_rl/observability/codec.py +119 -0
  15. risorius_rl/observability/config.py +118 -0
  16. risorius_rl/observability/passive_env.py +197 -0
  17. risorius_rl/observability/recording.py +458 -0
  18. risorius_rl/observability/session.py +538 -0
  19. risorius_rl/observability/spool.py +67 -0
  20. risorius_rl/observability/telemetry.py +222 -0
  21. risorius_rl/observability/transport.py +375 -0
  22. risorius_rl/py.typed +0 -0
  23. risorius_rl/pytorch.py +3 -0
  24. risorius_rl/runner.py +6 -0
  25. risorius_rl/runtime.py +3 -0
  26. risorius_rl/storage/__init__.py +1 -0
  27. risorius_rl/storage/datasets.py +84 -0
  28. risorius_rl/storage/evidence.py +217 -0
  29. risorius_rl/storage/files.py +183 -0
  30. risorius_rl/user_cli/__init__.py +1 -0
  31. risorius_rl/user_cli/auth.py +260 -0
  32. risorius_rl/user_cli/main.py +168 -0
  33. risorius_rl/user_cli/records.py +130 -0
  34. risorius_rl-0.15.1.dist-info/METADATA +78 -0
  35. risorius_rl-0.15.1.dist-info/RECORD +39 -0
  36. risorius_rl-0.15.1.dist-info/WHEEL +5 -0
  37. risorius_rl-0.15.1.dist-info/entry_points.txt +3 -0
  38. risorius_rl-0.15.1.dist-info/licenses/SDK-LICENSE +21 -0
  39. risorius_rl-0.15.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,22 @@
1
+ """Record results without exposing platform or AWS credentials to research code."""
2
+
3
+ from typing import TYPE_CHECKING
4
+
5
+ if TYPE_CHECKING:
6
+ from risorius_rl.execution.run import Run as Run
7
+ from risorius_rl.observability.session import RecordingSession as RecordingSession
8
+
9
+ __all__ = ["Run", "RecordingSession"]
10
+
11
+
12
+ def __getattr__(name):
13
+ # Recording-only imports must not load execution or cloud management clients.
14
+ if name == "Run":
15
+ from risorius_rl.execution.run import Run
16
+
17
+ return Run
18
+ if name == "RecordingSession":
19
+ from risorius_rl.observability.session import RecordingSession
20
+
21
+ return RecordingSession
22
+ raise AttributeError(name)
risorius_rl/cli.py ADDED
@@ -0,0 +1,272 @@
1
+ """Project-scoped research submission using the current AWS profile (SigV4)."""
2
+
3
+ import argparse
4
+ import json
5
+ import os
6
+ import re
7
+ import uuid
8
+ from pathlib import Path
9
+ from urllib.parse import urlparse
10
+
11
+ import boto3
12
+ import httpx
13
+ import yaml
14
+ from botocore.auth import SigV4Auth
15
+ from botocore.awsrequest import AWSRequest
16
+
17
+
18
+ def request(method, path, body=None, *, endpoint=None):
19
+ endpoint = (endpoint or os.environ.get("RL_API_URL", "")).rstrip("/")
20
+ parsed = urlparse(endpoint)
21
+ if (
22
+ parsed.scheme != "https"
23
+ or not re.fullmatch(
24
+ r"[a-z0-9]+\.execute-api\.ap-northeast-2\.amazonaws\.com", parsed.netloc
25
+ )
26
+ or parsed.path
27
+ ):
28
+ raise ValueError("Set RL_API_URL to the platform's HTTPS API Gateway endpoint")
29
+ credentials = boto3.Session().get_credentials()
30
+ if credentials is None:
31
+ raise ValueError("Authenticate your AWS profile first")
32
+ content = json.dumps(body, allow_nan=False) if body is not None else ""
33
+ signed = AWSRequest(
34
+ method=method,
35
+ url=endpoint + path,
36
+ data=content,
37
+ headers={"Content-Type": "application/json"},
38
+ )
39
+ SigV4Auth(credentials.get_frozen_credentials(), "execute-api", "ap-northeast-2").add_auth(
40
+ signed
41
+ )
42
+ with httpx.Client(timeout=30, trust_env=False) as client:
43
+ response = client.request(
44
+ method, endpoint + path, content=content, headers=dict(signed.headers)
45
+ )
46
+ if response.is_error:
47
+ raise RuntimeError(f"Platform returned {response.status_code}: {response.text}")
48
+ return response.json()
49
+
50
+
51
+ def main():
52
+ parser = argparse.ArgumentParser(description=__doc__)
53
+ commands = parser.add_subparsers(dest="command", required=True)
54
+ local = commands.add_parser("local")
55
+ local.add_argument("spec", type=Path)
56
+ local.add_argument("--output", type=Path, default=Path("artifacts/local"))
57
+ local.add_argument(
58
+ "--gpu-metrics",
59
+ action="store_true",
60
+ help="Sample visible NVIDIA GPUs; does not allocate a GPU",
61
+ )
62
+ prepare_command = commands.add_parser(
63
+ "workspace-prepare", help="Prepare a private run session and an owner approval request"
64
+ )
65
+ prepare_command.add_argument("spec", type=Path)
66
+ prepare_command.add_argument("--workspace", required=True)
67
+ prepare_command.add_argument("--session", type=Path, required=True)
68
+ workspace = commands.add_parser(
69
+ "workspace-run", help="Record training in an existing GPU workspace"
70
+ )
71
+ workspace.add_argument("session", type=Path)
72
+ workspace.add_argument("--url", required=True)
73
+ workspace.add_argument("--ca-file", type=Path, required=True)
74
+ external_prepare = commands.add_parser(
75
+ "external-prepare", help="Prepare recording on an external GPU"
76
+ )
77
+ external_prepare.add_argument("spec", type=Path)
78
+ external_prepare.add_argument("--provider", required=True)
79
+ external_prepare.add_argument("--duration-minutes", type=int, default=60)
80
+ external_prepare.add_argument("--session", type=Path, required=True)
81
+ external_run = commands.add_parser(
82
+ "external-run", help="Run with an approved external recording session"
83
+ )
84
+ external_run.add_argument("session", type=Path)
85
+ external_run.add_argument("--url", required=True)
86
+ external_run.add_argument("--ca-file", type=Path)
87
+ submit = commands.add_parser("submit")
88
+ submit.add_argument("spec", type=Path)
89
+ submit.add_argument("--image", required=True)
90
+ submit.add_argument("--run-id", default=None)
91
+ register = commands.add_parser("register-image")
92
+ register.add_argument("--project", required=True)
93
+ register.add_argument("--image", required=True)
94
+ register.add_argument("--commit", required=True)
95
+ register.add_argument("--backend", choices=("job", "ray"), default="job")
96
+ build = commands.add_parser("build", help="Package an approved GitHub source commit")
97
+ build.add_argument("--project", required=True)
98
+ build.add_argument("--commit", required=True)
99
+ build.add_argument("--request-id", default=None)
100
+ build.add_argument("--dependencies", choices=("project", "cpu", "ray"), default="project")
101
+ builds = commands.add_parser("builds", help="List the latest source builds")
102
+ builds.add_argument("--project", required=True)
103
+ for name in ("build-get", "build-cancel"):
104
+ command = commands.add_parser(name)
105
+ command.add_argument("--project", required=True)
106
+ command.add_argument("build_id")
107
+ for name in ("get", "cancel", "result", "list", "recordings"):
108
+ command = commands.add_parser(name)
109
+ command.add_argument("--project", required=True)
110
+ if name != "list":
111
+ command.add_argument("run_id")
112
+ artifact = commands.add_parser("download")
113
+ artifact.add_argument("--project", required=True)
114
+ artifact.add_argument("run_id")
115
+ artifact.add_argument("name")
116
+ artifact.add_argument("--output", type=Path, required=True)
117
+ recording = commands.add_parser(
118
+ "download-recording", help="Download committed chunks with SHA-256 verification"
119
+ )
120
+ recording.add_argument("--project", required=True)
121
+ recording.add_argument("run_id")
122
+ recording.add_argument("--stream", required=True)
123
+ recording.add_argument("--output", type=Path, required=True)
124
+ bundle = commands.add_parser(
125
+ "download-run", help="Download all recording streams, summaries and runtime logs"
126
+ )
127
+ bundle.add_argument("--project", required=True)
128
+ bundle.add_argument("run_id")
129
+ bundle.add_argument("--output", type=Path, required=True)
130
+ args = parser.parse_args()
131
+ try:
132
+ if args.command == "external-prepare":
133
+ from risorius_rl.execution.attached import prepare
134
+
135
+ result = prepare(
136
+ yaml.safe_load(args.spec.read_text()),
137
+ None,
138
+ args.session,
139
+ provider=args.provider,
140
+ duration_minutes=args.duration_minutes,
141
+ )
142
+ print(json.dumps(result, indent=2))
143
+ return
144
+ if args.command == "external-run":
145
+ from risorius_rl.execution.attached import execute_attached
146
+
147
+ result, code = execute_attached(
148
+ args.session, args.url, args.ca_file, backend="external"
149
+ )
150
+ print(json.dumps(result, indent=2))
151
+ if code:
152
+ parser.exit(code, "External execution did not succeed\n")
153
+ return
154
+ if args.command == "workspace-prepare":
155
+ from risorius_rl.execution.workspace import prepare
156
+
157
+ result = prepare(yaml.safe_load(args.spec.read_text()), args.workspace, args.session)
158
+ print(json.dumps(result, indent=2))
159
+ return
160
+ if args.command == "workspace-run":
161
+ from risorius_rl.execution.workspace import execute_workspace
162
+
163
+ result, code = execute_workspace(args.session, args.url, args.ca_file)
164
+ print(json.dumps(result, indent=2))
165
+ if code:
166
+ parser.exit(code, "Workspace execution did not succeed\n")
167
+ return
168
+ if args.command in {"local", "submit"}:
169
+ spec = yaml.safe_load(args.spec.read_text())
170
+ if args.command == "local":
171
+ from risorius_rl.execution.local import execute_local
172
+
173
+ local_result, code = execute_local(spec, args.output, gpu_metrics=args.gpu_metrics)
174
+ if code:
175
+ parser.exit(code, f"Local execution failed; evidence: {local_result.parent}\n")
176
+ print(str(local_result))
177
+ return
178
+ run_id = args.run_id or uuid.uuid4().hex
179
+ print(f"Run ID: {run_id} (reuse --run-id if the response is lost)", flush=True)
180
+ result = request(
181
+ "POST",
182
+ f"/v1/projects/{spec['project']}/runs",
183
+ {"run_id": run_id, "image": args.image, "spec": spec},
184
+ )
185
+ else:
186
+ base = f"/v1/projects/{args.project}"
187
+ if args.command == "build":
188
+ request_id = args.request_id or uuid.uuid4().hex
189
+ print(
190
+ f"Build request ID: {request_id} (reuse --request-id within 5 minutes if the response is lost)",
191
+ flush=True,
192
+ )
193
+ result = request(
194
+ "POST",
195
+ base + "/builds",
196
+ {
197
+ "commit": args.commit,
198
+ "request_id": request_id,
199
+ "dependencies": args.dependencies,
200
+ },
201
+ )
202
+ elif args.command == "builds":
203
+ result = request("GET", base + "/builds")
204
+ elif args.command in {"build-get", "build-cancel"}:
205
+ path = base + "/builds/" + args.build_id
206
+ result = (
207
+ request("POST", path + "/cancel", {})
208
+ if args.command == "build-cancel"
209
+ else request("GET", path)
210
+ )
211
+ elif args.command == "register-image":
212
+ result = request(
213
+ "POST",
214
+ base + "/images",
215
+ {
216
+ "image": args.image,
217
+ "commit": args.commit,
218
+ **({"backend": "ray"} if args.backend == "ray" else {}),
219
+ },
220
+ )
221
+ elif args.command == "list":
222
+ result = request("GET", base + "/runs")
223
+ elif args.command == "download":
224
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", args.name):
225
+ raise ValueError("Invalid artifact name")
226
+ result = request("GET", base + "/runs/" + args.run_id + "/artifacts/" + args.name)
227
+ with httpx.Client(timeout=60, trust_env=False) as client:
228
+ response = client.get(result["url"])
229
+ response.raise_for_status()
230
+ with args.output.open("xb") as target:
231
+ target.write(response.content)
232
+ result = {"saved": str(args.output)}
233
+ elif args.command == "download-run":
234
+ from risorius_rl.storage.evidence import download_run
235
+
236
+ result = download_run(request, base + "/runs/" + args.run_id, args.output)
237
+ elif args.command == "download-recording":
238
+ prefix = base + "/runs/" + args.run_id
239
+ snapshot = request("GET", prefix + "/recordings")
240
+ stream = next(
241
+ (s for s in snapshot["streams"] if s["stream_id"] == args.stream), None
242
+ )
243
+ if stream is None or "manifest" not in stream:
244
+ raise ValueError(
245
+ "Recording stream is unavailable or registration is incomplete"
246
+ )
247
+ from risorius_rl.storage.evidence import download_stream
248
+
249
+ with httpx.Client(timeout=60, trust_env=False) as client:
250
+ download_stream(request, client, prefix, stream, args.output)
251
+ (args.output / "snapshot.json").write_text(json.dumps(snapshot, indent=2))
252
+ result = {
253
+ "saved": str(args.output),
254
+ "chunks": stream["next_chunk"],
255
+ "closed": stream["state"]["closed"],
256
+ }
257
+ else:
258
+ path = base + "/runs/" + args.run_id
259
+ if args.command in {"cancel", "result", "recordings"}:
260
+ path += "/" + args.command
261
+ result = request(
262
+ "POST" if args.command == "cancel" else "GET",
263
+ path,
264
+ {} if args.command == "cancel" else None,
265
+ )
266
+ print(json.dumps(result, indent=2, ensure_ascii=False))
267
+ except (ValueError, RuntimeError, httpx.HTTPError) as error:
268
+ parser.exit(1, str(error) + "\n")
269
+
270
+
271
+ if __name__ == "__main__":
272
+ main()
@@ -0,0 +1 @@
1
+ """Run lifecycle and managed subprocess/Ray execution."""
@@ -0,0 +1,207 @@
1
+ """Prepare a scoped approval and run research on an already available machine."""
2
+
3
+ import base64
4
+ import hashlib
5
+ import json
6
+ import os
7
+ import re
8
+ import secrets
9
+ import signal
10
+ import stat
11
+ import subprocess
12
+ import sys
13
+ import time
14
+ import uuid
15
+ from pathlib import Path
16
+ from urllib.parse import urlparse
17
+
18
+ import httpx
19
+
20
+ from risorius_rl.execution.local import source_identity
21
+ from risorius_rl.observability.codec import json_bytes
22
+
23
+
24
+ def prepare(spec, workspace_id, session_file, *, provider=None, duration_minutes=60):
25
+ backend = "external" if provider is not None else "workspace"
26
+ if backend == "workspace":
27
+ workspace_id = str(uuid.UUID(workspace_id))
28
+ elif (
29
+ not isinstance(provider, str)
30
+ or not re.fullmatch(r"[a-z][a-z0-9-]{0,31}", provider)
31
+ or type(duration_minutes) is not int
32
+ or not 1 <= duration_minutes <= 1440
33
+ ):
34
+ raise ValueError("Use a provider name and a recording duration of 1..1440 minutes")
35
+ if not isinstance(spec, dict) or set(spec) - {
36
+ "project",
37
+ "kind",
38
+ "entrypoint",
39
+ "parameters",
40
+ "resources",
41
+ "backend",
42
+ }:
43
+ raise ValueError("Workspace runs support project, kind, entrypoint and parameters only")
44
+ if spec.get("backend", "job") != "job":
45
+ raise ValueError("Prepare a single-process job spec")
46
+ # Local validation is for usability; the Gateway independently validates the approved payload.
47
+ if not re.fullmatch(r"[a-z][a-z0-9-]{0,47}", spec.get("project", "")):
48
+ raise ValueError("Invalid project")
49
+ if spec.get("kind") not in {"rollout", "world-model"} or not isinstance(
50
+ spec.get("parameters"), dict
51
+ ):
52
+ raise ValueError("Use rollout/world-model and a parameters dictionary")
53
+ argv = spec.get("entrypoint")
54
+ if (
55
+ not isinstance(argv, list)
56
+ or not argv
57
+ or not all(isinstance(arg, str) and arg and "\x00" not in arg for arg in argv)
58
+ ):
59
+ raise ValueError("Use an entrypoint argument list")
60
+ spec = {**spec, "backend": backend, "resources": backend}
61
+ payload = json_bytes({"spec": spec, "source": source_identity()})
62
+ if len(payload) > 32 * 1024:
63
+ raise ValueError("Workspace payload exceeds 32 KiB")
64
+ token = secrets.token_urlsafe(32)
65
+ run_id = uuid.uuid4().hex
66
+ session = {"run_id": run_id, "token": token, "payload": payload.decode()}
67
+ fd = os.open(session_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
68
+ with os.fdopen(fd, "w") as output:
69
+ json.dump(session, output)
70
+ return {
71
+ "run_id": run_id,
72
+ "session_file": str(Path(session_file).resolve()),
73
+ "approval": {
74
+ "method": "PUT",
75
+ "path": (
76
+ f"/v1/projects/{spec['project']}/workspaces/{workspace_id}/runs/{run_id}/grant"
77
+ if backend == "workspace"
78
+ else f"/v1/projects/{spec['project']}/external-runs/{run_id}/grant"
79
+ ),
80
+ "body": {
81
+ "tokenSha256": hashlib.sha256(token.encode()).hexdigest(),
82
+ "payloadSha256": hashlib.sha256(payload).hexdigest(),
83
+ **(
84
+ {"provider": provider, "durationMinutes": duration_minutes}
85
+ if backend == "external"
86
+ else {}
87
+ ),
88
+ },
89
+ },
90
+ }
91
+
92
+
93
+ def execute_attached(session_file, url, ca_file, *, backend="workspace"):
94
+ """A private session file is a run capability, never a general AWS or workspace key."""
95
+ if backend not in {"workspace", "external"}:
96
+ raise ValueError("Invalid execution backend")
97
+ parsed = urlparse(url)
98
+ if (
99
+ parsed.scheme != "https"
100
+ or not parsed.hostname
101
+ or parsed.username
102
+ or parsed.password
103
+ or parsed.query
104
+ or parsed.fragment
105
+ or parsed.path
106
+ not in ({"", "/", "/recording", "/recording/"} if backend == "external" else {"", "/"})
107
+ ):
108
+ raise ValueError("Use the approved HTTPS recording Gateway origin")
109
+ url = url.rstrip("/")
110
+ fd = os.open(session_file, os.O_RDONLY | os.O_NOFOLLOW)
111
+ with os.fdopen(fd) as source:
112
+ meta = os.fstat(source.fileno())
113
+ if (
114
+ not stat.S_ISREG(meta.st_mode)
115
+ or meta.st_uid != os.getuid()
116
+ or meta.st_mode & 0o077
117
+ or meta.st_size > 64 * 1024
118
+ ):
119
+ raise ValueError("Session file must be owned by you with mode 0600 and <=64 KiB")
120
+ session = json.load(source)
121
+ run_id, token, payload = session["run_id"], session["token"], session["payload"]
122
+ if not re.fullmatch(r"[0-9a-f]{32}", run_id) or not re.fullmatch(r"[A-Za-z0-9_-]{43}", token):
123
+ raise ValueError("Invalid workspace session")
124
+ request_payload = json.loads(payload)
125
+ if request_payload["spec"].get("backend") != backend:
126
+ raise ValueError("Session belongs to a different execution backend")
127
+ with httpx.Client(
128
+ verify=str(ca_file.resolve()) if ca_file else True,
129
+ timeout=15,
130
+ trust_env=False,
131
+ headers={"Authorization": "Bearer " + token},
132
+ follow_redirects=False,
133
+ ) as client:
134
+
135
+ def send(path, content, *, retry=True):
136
+ for attempt in range(3):
137
+ try:
138
+ response = client.post(url + path, content=content)
139
+ # The ingress emits this header only before forwarding the request.
140
+ rejected = (
141
+ backend == "external"
142
+ and response.status_code == 429
143
+ and response.headers.get("Retry-After") == "5"
144
+ )
145
+ if rejected and attempt < 2:
146
+ time.sleep(5)
147
+ continue
148
+ if response.status_code in {429, 500, 502, 503, 504} and retry and attempt < 2:
149
+ time.sleep(attempt + 1)
150
+ continue
151
+ # Do not include a response body or session contents in error logs.
152
+ if response.status_code != 200:
153
+ raise RuntimeError(
154
+ f"Workspace Gateway returned HTTP {response.status_code}"
155
+ )
156
+ return response.json()
157
+ except httpx.TransportError:
158
+ if not retry or attempt == 2:
159
+ raise RuntimeError(
160
+ "Workspace Gateway transport failed; execution state may be unknown"
161
+ ) from None
162
+ time.sleep(attempt + 1)
163
+ raise RuntimeError("Workspace Gateway did not acknowledge request")
164
+
165
+ execution = send(f"/{backend}-runs/{run_id}", payload.encode())
166
+ if (
167
+ execution["spec"] != request_payload["spec"]
168
+ or execution["source"] != request_payload["source"]
169
+ ):
170
+ raise RuntimeError("Gateway execution differs from prepared payload")
171
+ # Only a definite ingress rejection is safe to retry for a one-time start.
172
+ send(f"/worker/{run_id}/{backend}-start", b"{}", retry=False)
173
+ env = {key: value for key, value in os.environ.items() if not key.startswith("RL_")}
174
+ env.update(
175
+ RL_RUN_ID=run_id,
176
+ RL_PROJECT_ID=execution["spec"]["project"],
177
+ RL_RUN_TOKEN=token,
178
+ RL_WORKER_URL=url,
179
+ RL_RECORDING_ENABLED="1",
180
+ RL_EXECUTION_B64=base64.b64encode(json_bytes(execution["spec"])).decode(),
181
+ PATH=str(Path(sys.executable).parent) + os.pathsep + os.environ.get("PATH", ""),
182
+ )
183
+ if ca_file:
184
+ env["RL_CA_FILE"] = str(ca_file.resolve())
185
+ if execution["image"]:
186
+ env["RL_IMAGE_DIGEST"] = execution["image"]
187
+ if execution["source"]["commit"]:
188
+ env["RL_SOURCE_COMMIT"] = execution["source"]["commit"]
189
+ print(f"{backend.capitalize()} run: {run_id}", file=sys.stderr, flush=True)
190
+ code = 1
191
+ try:
192
+ with subprocess.Popen(
193
+ [sys.executable, "-m", "risorius_rl.execution.runner"],
194
+ env=env,
195
+ start_new_session=True,
196
+ ) as process:
197
+ try:
198
+ code = process.wait()
199
+ except KeyboardInterrupt:
200
+ process.send_signal(signal.SIGTERM)
201
+ process.wait()
202
+ code = 130
203
+ finally:
204
+ outcome = send(f"/worker/{run_id}/{backend}-finish", json_bytes({"exit_code": code}))
205
+ return {"run_id": run_id, "state": outcome["state"]}, 0 if code == 0 and outcome[
206
+ "state"
207
+ ] == "succeeded" else 1
@@ -0,0 +1,119 @@
1
+ """Run research code on the current machine and retain local execution evidence.
2
+
3
+ This path neither allocates resources nor authenticates an execution to the server.
4
+ Source metadata is diagnostic and must never be treated as a trusted build identity.
5
+ """
6
+
7
+ import base64
8
+ import hashlib
9
+ import json
10
+ import os
11
+ import signal
12
+ import subprocess
13
+ import sys
14
+ import time
15
+ import uuid
16
+ from pathlib import Path
17
+
18
+
19
+ def source_identity() -> dict:
20
+ """Capture HEAD and dirty state without exporting URLs, paths or source contents."""
21
+ try:
22
+ commit = subprocess.run(
23
+ ["git", "rev-parse", "--verify", "HEAD"],
24
+ capture_output=True,
25
+ text=True,
26
+ check=True,
27
+ timeout=5,
28
+ ).stdout.strip()
29
+ status = subprocess.run(
30
+ ["git", "status", "--porcelain", "--untracked-files=normal"],
31
+ capture_output=True,
32
+ check=True,
33
+ timeout=5,
34
+ ).stdout
35
+ return {"commit": commit, "dirty": bool(status), "verified_by_platform": False}
36
+ except (OSError, subprocess.SubprocessError):
37
+ return {"commit": None, "dirty": None, "verified_by_platform": False}
38
+
39
+
40
+ def execute_local(spec: dict, output: Path, *, gpu_metrics: bool = False) -> tuple[Path, int]:
41
+ """Keep process outcome separate from a result written before a subsequent crash."""
42
+ if not isinstance(spec, dict) or spec.get("backend", "job") != "job":
43
+ raise ValueError("Local execution requires a job spec; use submit for managed Ray")
44
+ if not isinstance(spec.get("project"), str) or not spec["project"]:
45
+ raise ValueError("Set a project in the execution spec")
46
+ if spec.get("kind") not in {"rollout", "world-model"}:
47
+ raise ValueError("Use kind rollout or world-model")
48
+ if not isinstance(spec.get("parameters"), dict):
49
+ raise ValueError("Set parameters to a dictionary in the execution spec")
50
+ if (
51
+ not isinstance(spec.get("entrypoint"), list)
52
+ or not spec["entrypoint"]
53
+ or any(not isinstance(arg, str) or not arg or "\x00" in arg for arg in spec["entrypoint"])
54
+ ):
55
+ raise ValueError("Use a nonempty entrypoint argument list")
56
+ content = json.dumps(spec, allow_nan=False).encode()
57
+ source = source_identity()
58
+ run_id = uuid.uuid4().hex
59
+ directory = output.resolve() / run_id
60
+ directory.mkdir(parents=True, mode=0o700, exist_ok=False)
61
+ evidence = {
62
+ "schema_version": 1,
63
+ "mode": "local",
64
+ "run_id": run_id,
65
+ "project": spec["project"],
66
+ "source": source,
67
+ "spec_sha256": hashlib.sha256(content).hexdigest(),
68
+ "gpu_metrics_requested": gpu_metrics,
69
+ "started_at": time.time(),
70
+ "state": "running",
71
+ }
72
+
73
+ def save():
74
+ staging = directory / "execution.json.tmp"
75
+ staging.write_text(json.dumps(evidence, indent=2) + "\n")
76
+ staging.replace(directory / "execution.json")
77
+
78
+ save()
79
+ # A local run started inside a managed worker must not reuse its identity or token.
80
+ env = {key: value for key, value in os.environ.items() if not key.startswith("RL_")}
81
+ env.update(
82
+ RL_RUN_ID=run_id,
83
+ RL_PROJECT_ID=spec["project"],
84
+ RL_EXECUTION_B64=base64.b64encode(content).decode(),
85
+ RL_LOCAL_OUTPUT=str(directory),
86
+ RL_GPU_METRICS="1" if gpu_metrics else "0",
87
+ PATH=str(Path(sys.executable).parent) + os.pathsep + os.environ.get("PATH", ""),
88
+ )
89
+ if source["commit"]:
90
+ env["RL_SOURCE_COMMIT"] = source["commit"]
91
+ print(f"Local evidence: {directory}", file=sys.stderr, flush=True)
92
+ code = 1
93
+ try:
94
+ with subprocess.Popen(
95
+ [sys.executable, "-m", "risorius_rl.execution.runner"],
96
+ env=env,
97
+ start_new_session=True,
98
+ ) as process:
99
+ try:
100
+ code = process.wait()
101
+ except KeyboardInterrupt:
102
+ # The runner forwards TERM to its research process group and flushes its logs.
103
+ process.send_signal(signal.SIGTERM)
104
+ process.wait()
105
+ code = 130
106
+ except OSError as error:
107
+ evidence["error_type"] = type(error).__name__
108
+ finally:
109
+ result = directory / "result.json"
110
+ if code == 0 and not result.is_file():
111
+ code = 1
112
+ evidence["error_type"] = "MissingResult"
113
+ evidence.update(
114
+ state="succeeded" if code == 0 else "failed",
115
+ exit_code=code,
116
+ finished_at=time.time(),
117
+ )
118
+ save()
119
+ return result, code if code >= 0 else 128 - code