simulo-interfaces 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- simulo/interfaces/__init__.py +65 -0
- simulo/interfaces/exceptions.py +31 -0
- simulo/interfaces/ids.py +21 -0
- simulo/interfaces/platform/__init__.py +120 -0
- simulo/interfaces/platform/app.py +47 -0
- simulo/interfaces/platform/asset.py +15 -0
- simulo/interfaces/platform/callbacks.py +17 -0
- simulo/interfaces/platform/debug.py +31 -0
- simulo/interfaces/platform/domain.py +118 -0
- simulo/interfaces/platform/enums.py +49 -0
- simulo/interfaces/platform/runs.py +112 -0
- simulo/interfaces/platform/runtime.py +19 -0
- simulo/interfaces/platform/submit.py +538 -0
- simulo/interfaces/platform/volume.py +13 -0
- simulo/interfaces/py.typed +0 -0
- simulo/interfaces/runtime/__init__.py +50 -0
- simulo/interfaces/runtime/anomaly.py +195 -0
- simulo/interfaces/runtime/components.py +35 -0
- simulo/interfaces/runtime/env.py +27 -0
- simulo/interfaces/runtime/player.py +19 -0
- simulo/interfaces/runtime/policy.py +16 -0
- simulo/interfaces/runtime/scenario.py +13 -0
- simulo/interfaces/runtime/task.py +21 -0
- simulo/interfaces/runtime/tensors.py +12 -0
- simulo/interfaces/runtime/trainer.py +37 -0
- simulo_interfaces-0.1.0.dist-info/METADATA +166 -0
- simulo_interfaces-0.1.0.dist-info/RECORD +29 -0
- simulo_interfaces-0.1.0.dist-info/WHEEL +5 -0
- simulo_interfaces-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,538 @@
|
|
|
1
|
+
"""Submit / worker / recordings wire contract (Cloud Submit Wave, PR-0).
|
|
2
|
+
|
|
3
|
+
This is the **canonical, single source of truth** for the routes that carry a
|
|
4
|
+
package from the laptop client to the cloud, hand it to a worker, and return
|
|
5
|
+
logs/result/recordings to the client. It reconciles the design tracks in the
|
|
6
|
+
cloud-submit plan (`context/decisions.md`, Cloud Submit Wave) and lands
|
|
7
|
+
**before any consumer** — the control plane (client + `/internal/worker/*`
|
|
8
|
+
routes), the thin `simulo` client (`_client/submit_api.py`), and the worker
|
|
9
|
+
loop (`simulo-backend worker`) all implement against this module, not the
|
|
10
|
+
other way around. Implementation-free (Rule #11): route constants, size caps,
|
|
11
|
+
a wire dataclass, reserved-env constants, and semantics docstrings only — no
|
|
12
|
+
HTTP client, no server, no I/O.
|
|
13
|
+
|
|
14
|
+
**Zero changes to the existing job contract.** The four `GET /v1/jobs*`
|
|
15
|
+
routes and :class:`~simulo.interfaces.platform.runs.JobRecord` in
|
|
16
|
+
``runs.py`` are untouched by this module — ``POST /v1/jobs`` (defined here)
|
|
17
|
+
reuses the *same* route string as ``runs.JOBS_ROUTE`` (``"/v1/jobs"``; POST
|
|
18
|
+
creates, GET lists) and returns the *same* :class:`~simulo.interfaces.platform.runs.JobRecord`
|
|
19
|
+
shape. Scoping (org/project) is server-side and additive, exactly as
|
|
20
|
+
documented in ``runs.py``.
|
|
21
|
+
|
|
22
|
+
Trust zones (three bearer schemes on the wire; see the Cloud Submit Wave plan
|
|
23
|
+
architecture section for the full picture):
|
|
24
|
+
|
|
25
|
+
* **Client-facing `/v1/...`** — bearer = the user's Cognito JWT;
|
|
26
|
+
``require_active_org`` is enforced server-side. These are the routes a
|
|
27
|
+
human's ``simulo`` CLI/SDK calls.
|
|
28
|
+
* **Worker-facing `/internal/worker/...`** — bearer = a single static
|
|
29
|
+
``WORKER_TOKEN`` (constant-time compare on the server, never logged). Empty
|
|
30
|
+
server-side config disables every ``/internal/worker/*`` route
|
|
31
|
+
fail-closed — there is no "auth disabled" state for these routes.
|
|
32
|
+
* **Presigned URLs** (`download_url` / `upload_url` fields below) carry their
|
|
33
|
+
own auth (S3 SigV4 query params, or a control-plane-served one-time token
|
|
34
|
+
when running on the local storage backend) — a caller must send the
|
|
35
|
+
``Authorization`` bearer header *only* when the URL's host equals the API
|
|
36
|
+
host. Sending the API bearer to a presigned S3 host is both unnecessary and
|
|
37
|
+
a credential-leakage risk; this is a client/worker implementation rule, not
|
|
38
|
+
a wire shape, so it is documented here rather than encoded as a dataclass.
|
|
39
|
+
|
|
40
|
+
All timestamps are ISO 8601 UTC strings (e.g. ``"2026-07-03T12:00:00Z"``),
|
|
41
|
+
matching ``runs.py`` and the platform NFR contract. Errors use the
|
|
42
|
+
platform-standard shape: ``{"error": {"code": ..., "message": ..., "request_id": ...}}``.
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Client-facing: package submit + job create
|
|
47
|
+
|
|
48
|
+
* ``POST /v1/packages`` — register a content-addressed package before
|
|
49
|
+
uploading its bytes. Request body::
|
|
50
|
+
|
|
51
|
+
{"package_id": "...", "source_digest": "sha256:...",
|
|
52
|
+
"job_name": "train_cartpole", "archive_sha256": "sha256:...",
|
|
53
|
+
"archive_size": 123456}
|
|
54
|
+
|
|
55
|
+
Response ``200``::
|
|
56
|
+
|
|
57
|
+
{"package_id": "...", "upload": {"mode": "direct"}}
|
|
58
|
+
|
|
59
|
+
The ``upload`` envelope is forward-compatible: today the only mode is
|
|
60
|
+
``"direct"`` (the client ``PUT``s the tar to
|
|
61
|
+
:data:`PACKAGE_ARCHIVE_ROUTE_TEMPLATE` itself). A future ``"presigned"``
|
|
62
|
+
mode (client ``PUT``s straight to S3) is an **additive** variant of this
|
|
63
|
+
same envelope — the ``package_id`` field and the ``POST`` request body
|
|
64
|
+
never change.
|
|
65
|
+
|
|
66
|
+
* ``PUT /v1/packages/{package_id}/archive`` — the raw tar bytes (request
|
|
67
|
+
body is the archive, not JSON). Idempotent: if the content-addressed
|
|
68
|
+
``package_id`` already has an archive stored, a repeat ``PUT`` is a no-op
|
|
69
|
+
(``200``, bytes discarded) rather than an error — packaging the same
|
|
70
|
+
source twice is a client-retry, not a conflict. ``413`` (over
|
|
71
|
+
:data:`MAX_PACKAGE_BYTES`) and ``422`` (uploaded bytes' digest does not
|
|
72
|
+
match the registered ``archive_sha256``) are the only failure modes.
|
|
73
|
+
|
|
74
|
+
* ``POST /v1/jobs`` — reuses ``runs.JOBS_ROUTE`` (``"/v1/jobs"``; see the
|
|
75
|
+
module docstring above). Request body::
|
|
76
|
+
|
|
77
|
+
{"package_id": "...", "job_name": "train_cartpole", "args": {...}}
|
|
78
|
+
|
|
79
|
+
Response ``201`` — a :class:`~simulo.interfaces.platform.runs.JobRecord`
|
|
80
|
+
with ``status`` :attr:`~simulo.interfaces.platform.enums.JobStatus.QUEUED`.
|
|
81
|
+
**Idempotent replay:** a second ``POST`` with the same ``package_id`` while
|
|
82
|
+
a job for that package is still ``queued`` or ``running`` returns ``200``
|
|
83
|
+
with the *existing* record rather than creating a second job — a client
|
|
84
|
+
retry (e.g. after a dropped response) must not double-submit.
|
|
85
|
+
|
|
86
|
+
## Client-facing: recordings
|
|
87
|
+
|
|
88
|
+
* :data:`JOB_RECORDINGS_ROUTE_TEMPLATE` (``GET``) — paginated envelope of
|
|
89
|
+
:class:`RecordingRecord`, same pagination shape as ``GET /v1/jobs``::
|
|
90
|
+
|
|
91
|
+
{"items": [<RecordingRecord>, ...], "total": 2, "page": 1, "limit": 20, "pages": 1}
|
|
92
|
+
|
|
93
|
+
* :data:`JOB_RECORDING_DOWNLOAD_ROUTE_TEMPLATE` (``GET``) — resolves one
|
|
94
|
+
recording to a short-lived download link. Response::
|
|
95
|
+
|
|
96
|
+
{"url": "https://...", "expires_at": "2026-07-03T13:00:00Z"}
|
|
97
|
+
|
|
98
|
+
Org/project membership is re-checked on **this** request (Data Model §12)
|
|
99
|
+
— a recording's presence in a prior list response is not itself
|
|
100
|
+
authorization to download it. On staging this is a presigned S3 GET; on
|
|
101
|
+
the local storage backend it is a control-plane-served URL with its own
|
|
102
|
+
short-lived, single-use token.
|
|
103
|
+
|
|
104
|
+
## Worker-facing (`/internal/worker/*`)
|
|
105
|
+
|
|
106
|
+
* :data:`WORKER_CLAIM_ROUTE` (``POST``) — request ``{"worker_id": "..."}``.
|
|
107
|
+
Response ``200``::
|
|
108
|
+
|
|
109
|
+
{"job": <JobRecord>,
|
|
110
|
+
"package": {"download_url": "...", "tar_sha256": "sha256:...", "size_bytes": 123456},
|
|
111
|
+
"lease_seconds": 300,
|
|
112
|
+
"lease_id": "5e0c4a4e-..."}
|
|
113
|
+
|
|
114
|
+
or ``204`` with an empty body when there is no queued work. A claim is one
|
|
115
|
+
atomic ``SELECT ... FOR UPDATE SKIP LOCKED`` transition of exactly one
|
|
116
|
+
``queued`` job to ``running`` — plus a status event, ``attempts += 1``, and
|
|
117
|
+
a fresh lease. Two workers racing the same claim never both win.
|
|
118
|
+
|
|
119
|
+
``lease_id`` (lease-reaper wave, additive) is the **per-claim ownership
|
|
120
|
+
token**: a fresh server-generated opaque id stamped on the job row by this
|
|
121
|
+
claim. The worker must present it back on every subsequent job-scoped call
|
|
122
|
+
(heartbeat / logs / complete) in the :data:`WORKER_LEASE_HEADER` request
|
|
123
|
+
header — it is what distinguishes "the worker holding the CURRENT claim"
|
|
124
|
+
from "a worker that once held a claim on this job_id". A call whose
|
|
125
|
+
presented lease does not match the job's current one (the job was reaped
|
|
126
|
+
after a lease expiry and reclaimed — possibly by another worker) is
|
|
127
|
+
rejected ``409 lease_lost``; the losing worker must abort its subprocess
|
|
128
|
+
and stop reporting for that job.
|
|
129
|
+
|
|
130
|
+
**Lease expiry / reaper semantics:** the lease is the job's liveness
|
|
131
|
+
signal. A ``running`` job whose lease has expired is *reaped* by the
|
|
132
|
+
server (the reap runs inside the claim path, so any worker's next claim
|
|
133
|
+
heals the queue): while the job has attempts left it transitions back to
|
|
134
|
+
``queued`` (status event ``reason_code="lease_expired"``) for retry —
|
|
135
|
+
the next claim re-runs it with a fresh ``lease_id`` and ``attempts + 1``;
|
|
136
|
+
once attempts are exhausted it transitions terminally to ``failed``
|
|
137
|
+
(``reason_code="lease_expired_max_attempts"``). Expiry is enforced at
|
|
138
|
+
REAP time, not at heartbeat time: a heartbeat/complete presenting the
|
|
139
|
+
MATCHING lease of an expired-but-not-yet-reaped claim still succeeds
|
|
140
|
+
(the worker was merely slow — reviving is strictly better than killing
|
|
141
|
+
a live run), because the atomic ``lease_id`` swap at reclaim is what
|
|
142
|
+
fences out a stale worker, not the wall clock.
|
|
143
|
+
|
|
144
|
+
* :data:`WORKER_JOB_LOGS_ROUTE_TEMPLATE` (``POST``, ``?offset=N``) — request
|
|
145
|
+
body is raw log bytes (not JSON), capped at :data:`MAX_LOG_CHUNK_BYTES` per
|
|
146
|
+
call. ``offset`` is the byte offset the worker believes the server is at
|
|
147
|
+
(mirrors the client-side ``GET .../logs?offset=N`` semantics in
|
|
148
|
+
``runs.py``, from the writer's side):
|
|
149
|
+
|
|
150
|
+
- ``offset == <server's current total>`` — append normally.
|
|
151
|
+
- ``offset < <server's current total>`` — a duplicate resend (e.g. a
|
|
152
|
+
retried request whose first attempt actually landed); the server drops
|
|
153
|
+
the (already-applied) bytes and returns its current total — not an
|
|
154
|
+
error.
|
|
155
|
+
- ``offset > <server's current total>`` — a gap (a lost chunk in between);
|
|
156
|
+
``409`` ``log_offset_gap`` with the server's actual total so the worker
|
|
157
|
+
resyncs and resends from the correct offset.
|
|
158
|
+
|
|
159
|
+
Total accumulated log bytes for a job are capped at
|
|
160
|
+
:data:`MAX_LOG_TOTAL_BYTES`; once reached, further chunks are rejected
|
|
161
|
+
(job output past the cap is truncated, not the job).
|
|
162
|
+
|
|
163
|
+
Ownership: the request must carry the claim's ``lease_id`` in
|
|
164
|
+
:data:`WORKER_LEASE_HEADER` (the body is raw bytes, so the header is the
|
|
165
|
+
only channel). A stale lease is ``409 lease_lost`` — a reaped worker can
|
|
166
|
+
never append into the stream a later attempt now owns.
|
|
167
|
+
|
|
168
|
+
* :data:`WORKER_JOB_HEARTBEAT_ROUTE_TEMPLATE` (``POST``) — no request body
|
|
169
|
+
required; extends the claim lease. The request carries the claim's
|
|
170
|
+
``lease_id`` in :data:`WORKER_LEASE_HEADER`; a stale lease is ``409
|
|
171
|
+
lease_lost`` — the signal the losing worker uses to abort its subprocess
|
|
172
|
+
(the same abort plumbing the ``cancelled`` seam drives). Response
|
|
173
|
+
``{"cancelled": false}`` — the seam a future ``simulo cancel`` uses to
|
|
174
|
+
signal the worker to stop (always ``false`` today; no cancellation path
|
|
175
|
+
exists yet).
|
|
176
|
+
|
|
177
|
+
* :data:`WORKER_JOB_COMPLETE_ROUTE_TEMPLATE` (``POST``) — request::
|
|
178
|
+
|
|
179
|
+
{"status": "completed", "exit_code": 0,
|
|
180
|
+
"reason_code": null, "message": null, "result": {...}}
|
|
181
|
+
|
|
182
|
+
``status`` is :attr:`~simulo.interfaces.platform.enums.JobStatus.COMPLETED`
|
|
183
|
+
or :attr:`~simulo.interfaces.platform.enums.JobStatus.FAILED` — a terminal
|
|
184
|
+
status. ``reason_code`` and ``message`` are optional (used on failure);
|
|
185
|
+
``result`` is the job's result JSON object (present only on success,
|
|
186
|
+
capped at :data:`MAX_RESULT_BYTES`). The server validates the job's
|
|
187
|
+
current status is ``running`` (any other current status is ``409
|
|
188
|
+
invalid_transition`` — a job cannot complete twice) AND that the request's
|
|
189
|
+
:data:`WORKER_LEASE_HEADER` matches the job's current ``lease_id``
|
|
190
|
+
(mismatch is ``409 lease_lost`` — a worker whose lease was reaped cannot
|
|
191
|
+
complete, or overwrite the result of, a job another worker has since
|
|
192
|
+
claimed). On success the server best-effort flattens the accumulated log
|
|
193
|
+
chunks to durable storage; a flatten failure does not fail the
|
|
194
|
+
``complete`` call.
|
|
195
|
+
|
|
196
|
+
* :data:`WORKER_JOB_RECORDINGS_PRESIGN_ROUTE_TEMPLATE` (``POST``) — request
|
|
197
|
+
``{"size_bytes": 123456}``. Response::
|
|
198
|
+
|
|
199
|
+
{"recording_id": "...", "upload_url": "...", "storage_uri": "..."}
|
|
200
|
+
|
|
201
|
+
The worker then ``PUT``s the MCAP bytes directly to ``upload_url``.
|
|
202
|
+
|
|
203
|
+
* :data:`WORKER_JOB_RECORDINGS_ROUTE_TEMPLATE` (``POST``) — finalizes a
|
|
204
|
+
presigned upload once the worker's ``PUT`` has succeeded. Request is the
|
|
205
|
+
:class:`RecordingRecord` fields the worker knows about the recording it
|
|
206
|
+
just uploaded (``recording_id``, ``name``, ``digest_sha256``,
|
|
207
|
+
``schema_version``, ``profile``, and the optional episode-summary fields);
|
|
208
|
+
the server verifies the uploaded object exists (and, where the storage
|
|
209
|
+
backend supports it, matches ``digest_sha256``) before inserting the
|
|
210
|
+
``job_recordings`` row. Response ``201`` — the persisted
|
|
211
|
+
:class:`RecordingRecord`.
|
|
212
|
+
|
|
213
|
+
## Client-facing: models (trained-checkpoint retrieval)
|
|
214
|
+
|
|
215
|
+
The exact analogue of the recordings surface, for the ``.pt`` policy
|
|
216
|
+
checkpoints a training job leaves in its result's ``checkpoint_dir``
|
|
217
|
+
(Data Model §4.6 Model; ``best.pt`` / ``latest.pt`` — see the
|
|
218
|
+
``simulo.checkpoint.v1`` sidecar schema). Non-training jobs simply have no
|
|
219
|
+
models; the list is empty, never an error.
|
|
220
|
+
|
|
221
|
+
* :data:`JOB_MODELS_ROUTE_TEMPLATE` (``GET``) — paginated envelope of
|
|
222
|
+
:class:`ModelRecord`, same pagination shape as ``GET /v1/jobs``::
|
|
223
|
+
|
|
224
|
+
{"items": [<ModelRecord>, ...], "total": 2, "page": 1, "limit": 20, "pages": 1}
|
|
225
|
+
|
|
226
|
+
* :data:`JOB_MODEL_ROUTE_TEMPLATE` (``GET``) — one :class:`ModelRecord`
|
|
227
|
+
(Job.md "Retrieve Model Information Using CLI"). ``404 model_not_found``
|
|
228
|
+
for an unknown/foreign/malformed id — never 403 (anti-enumeration).
|
|
229
|
+
|
|
230
|
+
* :data:`JOB_MODEL_DOWNLOAD_ROUTE_TEMPLATE` (``GET``) — resolves one model
|
|
231
|
+
to a short-lived download link, ``{"url": ..., "expires_at": ...}`` —
|
|
232
|
+
identical semantics to the recording download route, including the
|
|
233
|
+
membership-recheck-on-THIS-request rule. The client verifies the
|
|
234
|
+
downloaded bytes against the record's ``digest_sha256`` (which is
|
|
235
|
+
REQUIRED for models, unlike recordings — the worker always hashes the
|
|
236
|
+
checkpoint before finalizing).
|
|
237
|
+
|
|
238
|
+
## Worker-facing: models (upload after ``complete``)
|
|
239
|
+
|
|
240
|
+
* :data:`WORKER_JOB_MODELS_PRESIGN_ROUTE_TEMPLATE` (``POST``) — request
|
|
241
|
+
``{"size_bytes": 123456}``. Response::
|
|
242
|
+
|
|
243
|
+
{"model_id": "...", "upload_url": "...", "storage_uri": "..."}
|
|
244
|
+
|
|
245
|
+
The worker then ``PUT``s the checkpoint bytes directly to ``upload_url``.
|
|
246
|
+
The stored object key is the Data Model §9 layout
|
|
247
|
+
(``.../models/{model_id}/model.bin``) — the ORIGINAL filename travels in
|
|
248
|
+
the finalize body's ``name`` and is what clients save the download as.
|
|
249
|
+
|
|
250
|
+
* :data:`WORKER_JOB_MODELS_ROUTE_TEMPLATE` (``POST``) — finalizes a
|
|
251
|
+
presigned model upload. Request: ``{"model_id": ..., "name": "best.pt",
|
|
252
|
+
"kind": "best", "digest_sha256": "sha256:...", "content_type":
|
|
253
|
+
"application/octet-stream"}``. ``kind`` is one of :data:`MODEL_KINDS`
|
|
254
|
+
(an unknown kind is stored as unlabelled, not rejected). The server
|
|
255
|
+
verifies the uploaded object exists and takes ``size_bytes`` from the
|
|
256
|
+
STORED object, never from the worker's claim. Response ``201`` — the
|
|
257
|
+
persisted :class:`ModelRecord`; an idempotent replay answers ``200``.
|
|
258
|
+
|
|
259
|
+
Like the recordings pair, presign/finalize are deliberately NOT
|
|
260
|
+
lease-gated server-side (the worker uploads models AFTER ``complete``,
|
|
261
|
+
when the lease is moot) — the anti-zombie invariant is worker-side: a
|
|
262
|
+
worker whose ``complete`` was 409-rejected (or whose heartbeat observed
|
|
263
|
+
``lease_lost``) must SKIP the model upload exactly as it skips the
|
|
264
|
+
recordings upload.
|
|
265
|
+
"""
|
|
266
|
+
|
|
267
|
+
from dataclasses import dataclass
|
|
268
|
+
from typing import Optional
|
|
269
|
+
|
|
270
|
+
from simulo.interfaces.ids import JobId, RecordingId, TrainedModelId
|
|
271
|
+
|
|
272
|
+
# --------------------------------------------------------------------------
|
|
273
|
+
# Client-facing routes (bearer = user JWT; require_active_org server-side)
|
|
274
|
+
# --------------------------------------------------------------------------
|
|
275
|
+
|
|
276
|
+
#: ``POST`` — register a content-addressed package before uploading its archive.
|
|
277
|
+
PACKAGES_ROUTE = "/v1/packages"
|
|
278
|
+
#: ``PUT`` — upload the raw tar bytes for a registered package. ``.format(package_id=...)``.
|
|
279
|
+
PACKAGE_ARCHIVE_ROUTE_TEMPLATE = "/v1/packages/{package_id}/archive"
|
|
280
|
+
|
|
281
|
+
#: ``GET`` — paginated envelope of :class:`RecordingRecord` for one job. ``.format(job_id=...)``.
|
|
282
|
+
JOB_RECORDINGS_ROUTE_TEMPLATE = "/v1/jobs/{job_id}/recordings"
|
|
283
|
+
#: ``GET`` — resolve one recording to a short-lived download link.
|
|
284
|
+
#: ``.format(job_id=..., recording_id=...)``.
|
|
285
|
+
JOB_RECORDING_DOWNLOAD_ROUTE_TEMPLATE = "/v1/jobs/{job_id}/recordings/{recording_id}/download"
|
|
286
|
+
|
|
287
|
+
#: ``GET`` — paginated envelope of :class:`ModelRecord` for one job. ``.format(job_id=...)``.
|
|
288
|
+
JOB_MODELS_ROUTE_TEMPLATE = "/v1/jobs/{job_id}/models"
|
|
289
|
+
#: ``GET`` — one :class:`ModelRecord`. ``.format(job_id=..., model_id=...)``.
|
|
290
|
+
JOB_MODEL_ROUTE_TEMPLATE = "/v1/jobs/{job_id}/models/{model_id}"
|
|
291
|
+
#: ``GET`` — resolve one model to a short-lived download link.
|
|
292
|
+
#: ``.format(job_id=..., model_id=...)``.
|
|
293
|
+
JOB_MODEL_DOWNLOAD_ROUTE_TEMPLATE = "/v1/jobs/{job_id}/models/{model_id}/download"
|
|
294
|
+
|
|
295
|
+
#: Note: ``POST /v1/jobs`` (create a job from a package) reuses
|
|
296
|
+
#: ``simulo.interfaces.platform.runs.JOBS_ROUTE`` — the *same* route string
|
|
297
|
+
#: as the existing ``GET /v1/jobs`` list endpoint (POST creates, GET lists).
|
|
298
|
+
#: There is deliberately no separate ``*_SUBMIT_ROUTE`` constant here — see
|
|
299
|
+
#: the module docstring's "Client-facing: package submit + job create" section.
|
|
300
|
+
|
|
301
|
+
# --------------------------------------------------------------------------
|
|
302
|
+
# Worker-facing routes (bearer = static WORKER_TOKEN, constant-time compare;
|
|
303
|
+
# empty server-side config disables every route below, fail-closed)
|
|
304
|
+
# --------------------------------------------------------------------------
|
|
305
|
+
|
|
306
|
+
#: ``POST`` — atomically claim the next queued job (``FOR UPDATE SKIP LOCKED``).
|
|
307
|
+
WORKER_CLAIM_ROUTE = "/internal/worker/claim"
|
|
308
|
+
#: ``POST`` — append raw log bytes at a byte offset (``?offset=N``). ``.format(job_id=...)``.
|
|
309
|
+
WORKER_JOB_LOGS_ROUTE_TEMPLATE = "/internal/worker/jobs/{job_id}/logs"
|
|
310
|
+
#: ``POST`` — extend the claim lease; response carries the ``cancelled`` seam.
|
|
311
|
+
#: ``.format(job_id=...)``.
|
|
312
|
+
WORKER_JOB_HEARTBEAT_ROUTE_TEMPLATE = "/internal/worker/jobs/{job_id}/heartbeat"
|
|
313
|
+
#: ``POST`` — transition a job to a terminal status with its exit code/result.
|
|
314
|
+
#: ``.format(job_id=...)``.
|
|
315
|
+
WORKER_JOB_COMPLETE_ROUTE_TEMPLATE = "/internal/worker/jobs/{job_id}/complete"
|
|
316
|
+
#: ``POST`` — obtain a presigned upload URL for a new recording. ``.format(job_id=...)``.
|
|
317
|
+
WORKER_JOB_RECORDINGS_PRESIGN_ROUTE_TEMPLATE = "/internal/worker/jobs/{job_id}/recordings/presign"
|
|
318
|
+
#: ``POST`` — finalize a presigned recording upload (inserts ``job_recordings`` row).
|
|
319
|
+
#: ``.format(job_id=...)``.
|
|
320
|
+
WORKER_JOB_RECORDINGS_ROUTE_TEMPLATE = "/internal/worker/jobs/{job_id}/recordings"
|
|
321
|
+
#: ``POST`` — obtain a presigned upload URL for a new model checkpoint. ``.format(job_id=...)``.
|
|
322
|
+
WORKER_JOB_MODELS_PRESIGN_ROUTE_TEMPLATE = "/internal/worker/jobs/{job_id}/models/presign"
|
|
323
|
+
#: ``POST`` — finalize a presigned model upload (inserts the ``models`` row).
|
|
324
|
+
#: ``.format(job_id=...)``.
|
|
325
|
+
WORKER_JOB_MODELS_ROUTE_TEMPLATE = "/internal/worker/jobs/{job_id}/models"
|
|
326
|
+
|
|
327
|
+
#: Request header carrying the claim's ``lease_id`` ownership token on every
|
|
328
|
+
#: job-scoped worker call (heartbeat / logs / complete). A header — not a body
|
|
329
|
+
#: field — because the logs route's body is raw bytes and heartbeat has no
|
|
330
|
+
#: body; one uniform channel for all three. A missing or mismatching value is
|
|
331
|
+
#: ``409 lease_lost`` (see the module docstring's claim section for the full
|
|
332
|
+
#: lease/reaper semantics). Recording presign/finalize are deliberately NOT
|
|
333
|
+
#: lease-gated: the worker uploads recordings AFTER ``complete`` — i.e. after
|
|
334
|
+
#: the job is terminal and the lease is moot.
|
|
335
|
+
WORKER_LEASE_HEADER = "X-Simulo-Lease-Id"
|
|
336
|
+
|
|
337
|
+
# --------------------------------------------------------------------------
|
|
338
|
+
# Size caps (security NFR — request-size limits per endpoint; see nfr SKILL)
|
|
339
|
+
# --------------------------------------------------------------------------
|
|
340
|
+
|
|
341
|
+
#: Maximum accepted size of a package archive (``PUT .../archive`` body). 413 over this.
|
|
342
|
+
MAX_PACKAGE_BYTES = 64 * 1024 * 1024 # 64 MiB
|
|
343
|
+
#: Maximum accepted size of a single **worker** log-upload call
|
|
344
|
+
#: (``POST`` :data:`WORKER_JOB_LOGS_ROUTE_TEMPLATE` body) — the write-side cap, 1 MiB.
|
|
345
|
+
#:
|
|
346
|
+
#: **Not the same constant as** ``simulo.interfaces.platform.runs.LOG_CHUNK_MAX_BYTES``
|
|
347
|
+
#: (64 KiB) — that is the pre-existing **client** read-side cap on a single
|
|
348
|
+
#: ``GET .../logs?offset=N`` response (``runs.py``). Both are now re-exported
|
|
349
|
+
#: from ``simulo.interfaces.platform``, so importing the wrong one silently
|
|
350
|
+
#: changes behavior by 16x: a worker that POSTs up to ``runs.LOG_CHUNK_MAX_BYTES``
|
|
351
|
+
#: per call under-uses its allowance; a client GET that budgeted for this
|
|
352
|
+
#: (larger) constant would be surprised the server never returns that much.
|
|
353
|
+
MAX_LOG_CHUNK_BYTES = 1 * 1024 * 1024 # 1 MiB
|
|
354
|
+
#: Maximum accumulated log bytes for one job, across every ``.../logs`` call.
|
|
355
|
+
#: Output past this cap is truncated; the job itself is not failed.
|
|
356
|
+
MAX_LOG_TOTAL_BYTES = 100 * 1024 * 1024 # 100 MiB
|
|
357
|
+
#: Maximum accepted size of a job's ``result`` JSON object
|
|
358
|
+
#: (``POST .../complete`` body's ``result`` field).
|
|
359
|
+
MAX_RESULT_BYTES = 1 * 1024 * 1024 # 1 MiB
|
|
360
|
+
#: Character caps on the ``POST .../complete`` body's human-facing fields —
|
|
361
|
+
#: the control plane's ``WorkerCompleteRequest`` schema enforces exactly these
|
|
362
|
+
#: (Pydantic ``max_length``). A worker MUST clamp before POSTing: an oversized
|
|
363
|
+
#: ``message`` 422-rejects the WHOLE terminal-status call, and a worker that
|
|
364
|
+
#: treats the 422 as best-effort leaves the job stuck ``running`` server-side
|
|
365
|
+
#: forever (found live in the first real-S3 staging run, 2026-07-04 — the
|
|
366
|
+
#: download-failure message embedded a ~1.9 KiB presigned URL).
|
|
367
|
+
MAX_COMPLETE_REASON_CODE_CHARS = 64
|
|
368
|
+
MAX_COMPLETE_MESSAGE_CHARS = 1024
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
372
|
+
class RecordingRecord:
|
|
373
|
+
"""One MCAP recording produced by a job — the wire schema returned by
|
|
374
|
+
:data:`JOB_RECORDINGS_ROUTE_TEMPLATE` and by the worker-facing recordings
|
|
375
|
+
finalize call (:data:`WORKER_JOB_RECORDINGS_ROUTE_TEMPLATE`).
|
|
376
|
+
|
|
377
|
+
``schema_version`` and ``profile`` describe the MCAP file itself (schema
|
|
378
|
+
versioning is owned by the `mcap-recording` SKILL — this dataclass only
|
|
379
|
+
carries the value through the wire, it does not define it).
|
|
380
|
+
The three episode-summary fields are optional because not every
|
|
381
|
+
recording is episodic (e.g. a raw rollout capture) and because a worker
|
|
382
|
+
may finalize a recording before its summary stats are computed.
|
|
383
|
+
"""
|
|
384
|
+
|
|
385
|
+
recording_id: RecordingId
|
|
386
|
+
job_id: JobId
|
|
387
|
+
name: str
|
|
388
|
+
size_bytes: int
|
|
389
|
+
digest_sha256: str
|
|
390
|
+
schema_version: str
|
|
391
|
+
profile: str
|
|
392
|
+
created_at: str
|
|
393
|
+
#: Number of episodes represented in this recording, if episodic.
|
|
394
|
+
n_episodes: Optional[int] = None
|
|
395
|
+
#: Mean per-episode return, if episodic and computed.
|
|
396
|
+
mean_episode_return: Optional[float] = None
|
|
397
|
+
#: Fraction of episodes that reached a success termination, if computed.
|
|
398
|
+
success_rate: Optional[float] = None
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
#: The checkpoint kinds a training job produces (``simulo.checkpoint.v1``:
|
|
402
|
+
#: ``best.pt`` = highest-mean-episode-reward snapshot, ``latest.pt`` = most
|
|
403
|
+
#: recent snapshot, kept for resume). The worker labels each uploaded model
|
|
404
|
+
#: with its kind; a kind outside this set is stored unlabelled, not rejected.
|
|
405
|
+
MODEL_KINDS = ("best", "latest")
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
409
|
+
class ModelRecord:
|
|
410
|
+
"""One trained-model artifact produced by a job (Data Model §4.6 Model) —
|
|
411
|
+
the wire schema returned by :data:`JOB_MODELS_ROUTE_TEMPLATE` /
|
|
412
|
+
:data:`JOB_MODEL_ROUTE_TEMPLATE` and by the worker-facing models
|
|
413
|
+
finalize call (:data:`WORKER_JOB_MODELS_ROUTE_TEMPLATE`).
|
|
414
|
+
|
|
415
|
+
``name`` is the artifact's ORIGINAL filename (``best.pt`` /
|
|
416
|
+
``latest.pt``) — the name a client saves the download under. The stored
|
|
417
|
+
object itself lives at the Data Model §9 key
|
|
418
|
+
(``.../models/{model_id}/model.bin``); the row's name/content_type are
|
|
419
|
+
what give the opaque object its user-facing identity. ``digest_sha256``
|
|
420
|
+
is always present (bare 64-hex): the worker hashes the checkpoint before
|
|
421
|
+
finalizing, and clients MUST verify the downloaded bytes against it.
|
|
422
|
+
"""
|
|
423
|
+
|
|
424
|
+
model_id: TrainedModelId
|
|
425
|
+
job_id: JobId
|
|
426
|
+
name: str
|
|
427
|
+
#: One of :data:`MODEL_KINDS`, or ``""`` when the kind is unknown.
|
|
428
|
+
kind: str
|
|
429
|
+
size_bytes: int
|
|
430
|
+
digest_sha256: str
|
|
431
|
+
content_type: str
|
|
432
|
+
created_at: str
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
# --------------------------------------------------------------------------
|
|
436
|
+
# Reserved runtime environment keys
|
|
437
|
+
# --------------------------------------------------------------------------
|
|
438
|
+
#
|
|
439
|
+
# The worker spawns the job subprocess (``simulo-backend run-package``) with
|
|
440
|
+
# the user's job ``args`` merged into its environment. This denylist is
|
|
441
|
+
# derived from a single principle — **runner-owned vs. user-tunable** — read
|
|
442
|
+
# directly off what ``simulo-backend``'s ``runner.py`` / ``run_store.py``
|
|
443
|
+
# actually do to ``os.environ`` (not guessed):
|
|
444
|
+
#
|
|
445
|
+
# * **Runner-owned** (reserved here): any ``SIMULO_*`` key the runner or
|
|
446
|
+
# run-store *sets* to identify the job/worker, resolve a filesystem path it
|
|
447
|
+
# then trusts, select execution mode, or authenticate outbound calls.
|
|
448
|
+
# ``runner.py`` sets ``SIMULO_MODE=execution`` unconditionally
|
|
449
|
+
# (``run_package()``); exports ``SIMULO_VOLUME_<name>`` / ``SIMULO_ASSET_<uri>``
|
|
450
|
+
# as resolved mount paths (``_volume_env_key`` / ``_asset_env_key`` /
|
|
451
|
+
# ``_setup_mounts``) — a user-supplied ``SIMULO_VOLUME_data=/etc`` would
|
|
452
|
+
# redirect a mount, the **sandbox-escape vector** and the highest-priority
|
|
453
|
+
# reservation here; prints the ``SIMULO_RESULT_JSON=`` stdout sentinel a
|
|
454
|
+
# parent process trusts as the job's result payload
|
|
455
|
+
# (``RESULT_SENTINEL_PREFIX``); sets ``SIMULO_RETRY_ATTEMPT`` per subprocess
|
|
456
|
+
# attempt so a child never starts its own retry loop; and ``run_store.py``
|
|
457
|
+
# resolves ``SIMULO_RUNS_ROOT`` to decide where run state is durably
|
|
458
|
+
# recorded. ``SIMULO_CHECKPOINT_DIR`` is the one checkpoint-envelope key
|
|
459
|
+
# that is also runner-owned (``_setup_checkpointing`` derives it from the
|
|
460
|
+
# job's *submission* identity — ``package_id`` + job name — so a
|
|
461
|
+
# user-supplied override would silently point training at another job's
|
|
462
|
+
# checkpoint state); ``SIMULO_API_TOKEN`` is the worker-bearer credential
|
|
463
|
+
# the job process would use to call back to the control plane once that
|
|
464
|
+
# lands (PR-5) — reserved proactively so the contract does not need a
|
|
465
|
+
# second pass. ``PATH`` / ``PYTHONPATH`` / ``HOME`` are reserved because
|
|
466
|
+
# the worker's ``pip install --target`` dependency layer depends on
|
|
467
|
+
# controlling ``PYTHONPATH``, and a job/worker identity key
|
|
468
|
+
# (``SIMULO_JOB_ID`` / ``SIMULO_PACKAGE_ID`` / ``SIMULO_WORKER_ID`` /
|
|
469
|
+
# ``SIMULO_WORKER_TOKEN`` / ``SIMULO_API_URL``) must not be spoofable by
|
|
470
|
+
# job ``args``. ``SIMULO_SHUTDOWN_GRACE_S`` (2026-07-07 shutdown watchdog)
|
|
471
|
+
# is the ORCHESTRATOR's own post-success grace-period knob — an operator
|
|
472
|
+
# env var, not a per-job one — so it is reserved defense-in-depth even
|
|
473
|
+
# though nothing in the current job-``args`` merge path would let a
|
|
474
|
+
# package set it today; a submitted package's manifest ``args`` must never
|
|
475
|
+
# be able to widen its own attempt child's kill grace.
|
|
476
|
+
#
|
|
477
|
+
# * **User-tunable** (deliberately NOT reserved): ``_setup_checkpointing``
|
|
478
|
+
# exports ``SIMULO_CHECKPOINT_EVERY`` / ``SIMULO_CHECKPOINT_KEEP_LAST`` /
|
|
479
|
+
# ``SIMULO_RESUME`` with ``os.environ.setdefault`` specifically so an
|
|
480
|
+
# explicit caller/job env value wins — these are job-tunable knobs by
|
|
481
|
+
# design, not runner secrets. That is why :data:`RESERVED_RUNTIME_ENV_KEYS`
|
|
482
|
+
# reserves the checkpoint **directory key only**
|
|
483
|
+
# (``SIMULO_CHECKPOINT_DIR``), never a ``SIMULO_CHECKPOINT_`` *prefix* — a
|
|
484
|
+
# prefix ban would also catch ``EVERY``/``KEEP_LAST``, the same class of
|
|
485
|
+
# bug this whole design avoids for ``SIMULO_EVAL_REWARD_TARGET`` (read by
|
|
486
|
+
# ``examples/cartpole_eval``; still legitimate user-supplied job env, and
|
|
487
|
+
# still not reserved by any key or prefix here).
|
|
488
|
+
#
|
|
489
|
+
# This is deliberately a **reserved-KEYS list, not a blanket `SIMULO_*` prefix
|
|
490
|
+
# ban.** Banning the whole prefix would make every future user-facing
|
|
491
|
+
# ``SIMULO_*`` job knob (the eval-target case, the checkpoint-tuning case, and
|
|
492
|
+
# whatever comes next) a breaking change to smuggle past this contract.
|
|
493
|
+
# Instead, :data:`RESERVED_RUNTIME_ENV_PREFIXES` reserves only the narrow
|
|
494
|
+
# sub-namespaces the runner/worker fully own — worker-internal identifiers,
|
|
495
|
+
# and the per-mount ``SIMULO_VOLUME_*`` / ``SIMULO_ASSET_*`` families, whose
|
|
496
|
+
# every member is runner-generated (there is no legitimate user-facing key
|
|
497
|
+
# under either prefix; every valid key that starts with them is a mount the
|
|
498
|
+
# runner itself created) — and :data:`RESERVED_RUNTIME_ENV_KEYS` reserves the
|
|
499
|
+
# specific mechanical/identity keys enumerated above. ``run_package()``
|
|
500
|
+
# (``simulo-backend``, PR-5) enforces this by skipping and warning on any
|
|
501
|
+
# user ``args`` key that matches either collection — it never raises, so a
|
|
502
|
+
# job with an (accidentally) colliding env key still runs, just without
|
|
503
|
+
# clobbering the runner's own environment.
|
|
504
|
+
|
|
505
|
+
#: Exact environment-variable names reserved for the runner/worker's own use
|
|
506
|
+
#: (see the "runner-owned vs. user-tunable" note above for the derivation of
|
|
507
|
+
#: every entry). A user-supplied job arg using one of these keys is skipped
|
|
508
|
+
#: (with a warning), never applied.
|
|
509
|
+
RESERVED_RUNTIME_ENV_KEYS = frozenset(
|
|
510
|
+
{
|
|
511
|
+
"PATH",
|
|
512
|
+
"PYTHONPATH",
|
|
513
|
+
"HOME",
|
|
514
|
+
"SIMULO_JOB_ID",
|
|
515
|
+
"SIMULO_PACKAGE_ID",
|
|
516
|
+
"SIMULO_API_URL",
|
|
517
|
+
"SIMULO_WORKER_ID",
|
|
518
|
+
"SIMULO_WORKER_TOKEN",
|
|
519
|
+
"SIMULO_RUNS_ROOT",
|
|
520
|
+
"SIMULO_CHECKPOINT_DIR",
|
|
521
|
+
"SIMULO_API_TOKEN",
|
|
522
|
+
"SIMULO_RESULT_JSON",
|
|
523
|
+
"SIMULO_MODE",
|
|
524
|
+
"SIMULO_RETRY_ATTEMPT",
|
|
525
|
+
"SIMULO_SHUTDOWN_GRACE_S",
|
|
526
|
+
}
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
#: Environment-variable name prefixes reserved for the runner/worker's own
|
|
530
|
+
#: internal namespaces — narrower than a blanket ``SIMULO_*`` ban (see the
|
|
531
|
+
#: module note above). ``SIMULO_VOLUME_*`` / ``SIMULO_ASSET_*`` are the mount
|
|
532
|
+
#: env keys ``runner._setup_mounts`` exports for every declared volume/asset
|
|
533
|
+
#: (the sandbox-escape vector: an unreserved ``SIMULO_VOLUME_<name>`` would
|
|
534
|
+
#: let a job arg redirect a mount path). ``SIMULO_EVAL_REWARD_TARGET`` (and
|
|
535
|
+
#: any other ``SIMULO_EVAL_*`` / ``SIMULO_CHECKPOINT_EVERY`` /
|
|
536
|
+
#: ``SIMULO_CHECKPOINT_KEEP_LAST`` / ``SIMULO_RESUME``) does not match any of
|
|
537
|
+
#: these prefixes and remains legitimate user-supplied job env.
|
|
538
|
+
RESERVED_RUNTIME_ENV_PREFIXES = ("SIMULO_WORKER_", "SIMULO_INTERNAL_", "SIMULO_VOLUME_", "SIMULO_ASSET_")
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""``VolumeProtocol`` — named, durable, writable cloud volume (Part B)."""
|
|
2
|
+
|
|
3
|
+
from typing import Protocol, runtime_checkable
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@runtime_checkable
|
|
7
|
+
class VolumeProtocol(Protocol):
|
|
8
|
+
"""Named, durable, writable volume. FORWARD."""
|
|
9
|
+
|
|
10
|
+
@classmethod
|
|
11
|
+
def from_name(cls, name: str, *, create_if_missing: bool = False) -> "VolumeProtocol": ...
|
|
12
|
+
@property
|
|
13
|
+
def name(self) -> str: ...
|
|
File without changes
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Part A — runtime/sim Protocols (torch-free structural mirrors of ``simulo.core``).
|
|
2
|
+
|
|
3
|
+
Tensor-library-agnostic ``typing.Protocol`` contracts for the SDK runtime
|
|
4
|
+
surface: Task, Policy, Trainer, Player, LearningEnv, Scenario, and the
|
|
5
|
+
observation/reward/termination components. Structural (no inheritance required)
|
|
6
|
+
and ``runtime_checkable`` for ``isinstance`` smoke checks. ``simulo-backend``'s
|
|
7
|
+
``simulo.core`` classes additionally declare *nominal* conformance (they list
|
|
8
|
+
these Protocols as base classes) so mypy drift-checks every implementation at
|
|
9
|
+
its definition site; other implementers remain free to conform structurally.
|
|
10
|
+
``PolicyExporterProtocol`` is the narrow optional export capability — see
|
|
11
|
+
``simulo.interfaces.runtime.trainer``.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from simulo.interfaces.runtime.anomaly import (
|
|
15
|
+
AnomalyDebugSessionProtocol,
|
|
16
|
+
AnomalyKind,
|
|
17
|
+
AnomalySignal,
|
|
18
|
+
DebugOnAnomaly,
|
|
19
|
+
)
|
|
20
|
+
from simulo.interfaces.runtime.components import (
|
|
21
|
+
ObservationComponentProtocol,
|
|
22
|
+
RewardComponentProtocol,
|
|
23
|
+
TerminationConditionProtocol,
|
|
24
|
+
)
|
|
25
|
+
from simulo.interfaces.runtime.env import LearningEnvProtocol
|
|
26
|
+
from simulo.interfaces.runtime.player import PlayerProtocol
|
|
27
|
+
from simulo.interfaces.runtime.policy import PolicyProtocol
|
|
28
|
+
from simulo.interfaces.runtime.scenario import ScenarioProtocol
|
|
29
|
+
from simulo.interfaces.runtime.task import TaskProtocol
|
|
30
|
+
from simulo.interfaces.runtime.tensors import TensorLike
|
|
31
|
+
from simulo.interfaces.runtime.trainer import PolicyExporterProtocol, TrainerProtocol
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"TensorLike",
|
|
35
|
+
"TaskProtocol",
|
|
36
|
+
"PolicyProtocol",
|
|
37
|
+
"TrainerProtocol",
|
|
38
|
+
"PolicyExporterProtocol",
|
|
39
|
+
"PlayerProtocol",
|
|
40
|
+
"LearningEnvProtocol",
|
|
41
|
+
"ScenarioProtocol",
|
|
42
|
+
"ObservationComponentProtocol",
|
|
43
|
+
"RewardComponentProtocol",
|
|
44
|
+
"TerminationConditionProtocol",
|
|
45
|
+
# anomaly capture (DebugOnAnomaly — the local half of Rule #10)
|
|
46
|
+
"AnomalyKind",
|
|
47
|
+
"AnomalySignal",
|
|
48
|
+
"DebugOnAnomaly",
|
|
49
|
+
"AnomalyDebugSessionProtocol",
|
|
50
|
+
]
|