simulo 0.23.0__tar.gz → 0.24.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {simulo-0.23.0/src/simulo.egg-info → simulo-0.24.0}/PKG-INFO +4 -4
- {simulo-0.23.0 → simulo-0.24.0}/pyproject.toml +7 -4
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/_secure_downloads.py +317 -25
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/app.py +54 -2
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/asset_api.py +32 -106
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/cli.py +347 -205
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/credentials.py +5 -1
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/preflight_render.py +9 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/submit_api.py +64 -15
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/view_session_api.py +2 -1
- {simulo-0.23.0 → simulo-0.24.0/src/simulo.egg-info}/PKG-INFO +4 -4
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo.egg-info/requires.txt +3 -3
- {simulo-0.23.0 → simulo-0.24.0}/MANIFEST.in +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/PYPI.md +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/setup.cfg +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/__init__.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/__init__.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/_entrypoint.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/_mounts.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/_runner.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/asset.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/asset_package.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/asset_pins.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/builtin_aliases.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/bundle.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/cancel_api.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/config.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/discovery.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/export_api.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/export_bundle.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/facades.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/http.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/identity_api.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/jobs_api.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/learning.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/login.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/mode.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/outputs.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/packaging.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/preflight_api.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/registry.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/runtime.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/runtime_display.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/seed_ref.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/stub.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/templates/__init__.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/templates/inference/app.py.tmpl +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/templates/inference/simuloignore.tmpl +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/templates/scenario/app.py.tmpl +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/templates/scenario/simuloignore.tmpl +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/templates/training/app.py.tmpl +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/templates/training/simuloignore.tmpl +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/view_fragment.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/_client/volume.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/callbacks.py +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo/py.typed +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo.egg-info/SOURCES.txt +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo.egg-info/dependency_links.txt +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo.egg-info/entry_points.txt +0 -0
- {simulo-0.23.0 → simulo-0.24.0}/src/simulo.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: simulo
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.24.0
|
|
4
4
|
Summary: Simulo SDK and CLI — define robotics simulation and training apps in Python and run them on the Simulo cloud.
|
|
5
5
|
Author-email: Simulo Team <team@simulo.ai>
|
|
6
6
|
License: BSD-3-Clause
|
|
@@ -15,12 +15,12 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
15
15
|
Classifier: Typing :: Typed
|
|
16
16
|
Requires-Python: >=3.11
|
|
17
17
|
Description-Content-Type: text/markdown
|
|
18
|
-
Requires-Dist: simulo-interfaces<0.
|
|
18
|
+
Requires-Dist: simulo-interfaces<0.19,>=0.18.0
|
|
19
19
|
Requires-Dist: mcap<2,>=1.3
|
|
20
20
|
Requires-Dist: defusedxml>=0.7.1
|
|
21
|
-
Requires-Dist: usd-core<27,>=
|
|
21
|
+
Requires-Dist: usd-core<27,>=26.5
|
|
22
22
|
Provides-Extra: assets
|
|
23
|
-
Requires-Dist: usd-core<27,>=
|
|
23
|
+
Requires-Dist: usd-core<27,>=26.5; extra == "assets"
|
|
24
24
|
Provides-Extra: dev
|
|
25
25
|
Requires-Dist: black==26.5.1; extra == "dev"
|
|
26
26
|
Requires-Dist: isort==8.0.1; extra == "dev"
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "simulo"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.24.0"
|
|
8
8
|
description = "Simulo SDK and CLI — define robotics simulation and training apps in Python and run them on the Simulo cloud."
|
|
9
9
|
readme = "PYPI.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -26,7 +26,7 @@ classifiers = [
|
|
|
26
26
|
# mcap: used by `simulo recordings` to read back a downloaded recording and
|
|
27
27
|
# verify it (message count) — MIT-licensed, deps only lz4+zstandard.
|
|
28
28
|
dependencies = [
|
|
29
|
-
"simulo-interfaces>=0.
|
|
29
|
+
"simulo-interfaces>=0.18.0,<0.19",
|
|
30
30
|
"mcap>=1.3,<2",
|
|
31
31
|
# Hardened XML parsing for URDF ingestion (USD Asset Catalogs, PR-12 fix
|
|
32
32
|
# loop 2, security NIT): stdlib xml.etree.ElementTree relies on
|
|
@@ -44,7 +44,7 @@ dependencies = [
|
|
|
44
44
|
# discover first). Still torch-free and still lazy-imported ONLY on the
|
|
45
45
|
# USD inspection path (`_client/asset_package.py::_load_pxr_usdutils`) —
|
|
46
46
|
# importing `simulo` itself never pays its cost.
|
|
47
|
-
"usd-core>=
|
|
47
|
+
"usd-core>=26.5,<27",
|
|
48
48
|
]
|
|
49
49
|
|
|
50
50
|
[project.optional-dependencies]
|
|
@@ -53,7 +53,7 @@ dependencies = [
|
|
|
53
53
|
# install above in 0.13.1, so this extra is now redundant (its dep is
|
|
54
54
|
# already installed unconditionally) but harmless — resolvers don't warn on
|
|
55
55
|
# an extra whose contents are already satisfied by the base requirement.
|
|
56
|
-
assets = ["usd-core>=
|
|
56
|
+
assets = ["usd-core>=26.5,<27"]
|
|
57
57
|
# black/isort/ruff/mypy pinned EXACT, not >=: these are rule-set tools whose
|
|
58
58
|
# defaults change between releases, so an unbounded lower bound lets CI silently
|
|
59
59
|
# resolve a newer release than a developer's environment has — "clean locally" and
|
|
@@ -141,6 +141,9 @@ module = ["pxr", "pxr.*"]
|
|
|
141
141
|
ignore_missing_imports = true
|
|
142
142
|
|
|
143
143
|
[tool.pytest.ini_options]
|
|
144
|
+
# Resolve the client from this checkout before any shared editable installation.
|
|
145
|
+
# This makes the documented pytest command exercise the checkout that owns tests/.
|
|
146
|
+
pythonpath = ["src"]
|
|
144
147
|
testpaths = ["tests"]
|
|
145
148
|
addopts = "--import-mode=importlib"
|
|
146
149
|
# A healthy hosted leg spent 36s before pytest and 63s in pytest, leaving roughly
|
|
@@ -110,9 +110,13 @@ before.
|
|
|
110
110
|
|
|
111
111
|
from __future__ import annotations
|
|
112
112
|
|
|
113
|
+
import ctypes
|
|
114
|
+
import errno
|
|
113
115
|
import os
|
|
114
116
|
import secrets
|
|
115
117
|
import stat as stat_module
|
|
118
|
+
import sys
|
|
119
|
+
import unicodedata
|
|
116
120
|
from pathlib import Path
|
|
117
121
|
from typing import Optional
|
|
118
122
|
|
|
@@ -137,6 +141,27 @@ class DestinationDirectoryError(OSError):
|
|
|
137
141
|
"""
|
|
138
142
|
|
|
139
143
|
|
|
144
|
+
def _retained_temp_error(
|
|
145
|
+
destination_display_path: Path, temp_name: str, exc: Exception, *, existing_final_preserved: bool = False
|
|
146
|
+
) -> DestinationDirectoryError:
|
|
147
|
+
"""Describe a possible temporary residue without trusting its current path."""
|
|
148
|
+
preservation = "Existing final entry was preserved. " if existing_final_preserved else ""
|
|
149
|
+
return DestinationDirectoryError(
|
|
150
|
+
f"{preservation}Download created private temporary entry {temp_name!r} in the originally selected "
|
|
151
|
+
f"destination directory {destination_display_path} before the failure. That directory and name may now "
|
|
152
|
+
"resolve elsewhere; locate and verify the original entry before cleanup. Do not delete a same-named "
|
|
153
|
+
"current path unless its identity is independently confirmed. "
|
|
154
|
+
f"Could not complete the download write: {exc}."
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _close_fd_quietly(fd: int) -> None:
|
|
159
|
+
try:
|
|
160
|
+
os.close(fd)
|
|
161
|
+
except OSError:
|
|
162
|
+
pass
|
|
163
|
+
|
|
164
|
+
|
|
140
165
|
#: ``True`` when this interpreter can perform openat-style,
|
|
141
166
|
#: directory-file-descriptor-relative operations for every primitive this
|
|
142
167
|
#: module needs. A platform property, checked once at import time — used by
|
|
@@ -164,6 +189,89 @@ SUPPORTS_DIR_FD = (
|
|
|
164
189
|
#: count for the same collision-on-``O_EXCL`` situation.
|
|
165
190
|
_TEMP_NAME_ATTEMPTS = 64
|
|
166
191
|
|
|
192
|
+
# Linux ``renameat2(..., RENAME_NOREPLACE)`` is one syscall: either the
|
|
193
|
+
# temporary entry becomes the final entry and the temporary name disappears,
|
|
194
|
+
# or an existing final entry is left untouched. ``link`` followed by
|
|
195
|
+
# ``unlink`` cannot offer that postcondition because a process death between
|
|
196
|
+
# those calls leaves both names behind. Keep the optional libc binding at
|
|
197
|
+
# module scope so unsupported libc/platform combinations are explicitly
|
|
198
|
+
# refused rather than silently falling back to replacement.
|
|
199
|
+
_AT_FDCWD = -100
|
|
200
|
+
_RENAME_NOREPLACE = 1
|
|
201
|
+
_RENAME_EXCL = 0x00000004
|
|
202
|
+
_renameat2 = None
|
|
203
|
+
_renameatx_np = None
|
|
204
|
+
if sys.platform.startswith("linux"):
|
|
205
|
+
_libc = ctypes.CDLL(None, use_errno=True)
|
|
206
|
+
_renameat2 = getattr(_libc, "renameat2", None)
|
|
207
|
+
if _renameat2 is not None:
|
|
208
|
+
_renameat2.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
|
|
209
|
+
_renameat2.restype = ctypes.c_int
|
|
210
|
+
elif sys.platform == "darwin":
|
|
211
|
+
_libc = ctypes.CDLL(None, use_errno=True)
|
|
212
|
+
_renameatx_np = getattr(_libc, "renameatx_np", None)
|
|
213
|
+
if _renameatx_np is not None:
|
|
214
|
+
_renameatx_np.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
|
|
215
|
+
_renameatx_np.restype = ctypes.c_int
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _rename_noreplace(
|
|
219
|
+
source: str,
|
|
220
|
+
destination: str,
|
|
221
|
+
*,
|
|
222
|
+
src_dir_fd: Optional[int] = None,
|
|
223
|
+
dst_dir_fd: Optional[int] = None,
|
|
224
|
+
) -> None:
|
|
225
|
+
"""Atomically publish *source* at absent *destination*, or fail closed.
|
|
226
|
+
|
|
227
|
+
Linux uses ``renameat2(RENAME_NOREPLACE)`` and macOS uses
|
|
228
|
+
``renameatx_np(RENAME_EXCL)``. Windows' native ``os.rename`` refuses an
|
|
229
|
+
existing destination. Other platforms deliberately refuse the
|
|
230
|
+
server-derived no-clobber path instead of using a two-name link/unlink
|
|
231
|
+
sequence or replacing an existing file. Explicit user file targets keep
|
|
232
|
+
using ``os.replace`` through ``replace_existing=True``.
|
|
233
|
+
"""
|
|
234
|
+
if os.name == "nt":
|
|
235
|
+
if src_dir_fd is not None or dst_dir_fd is not None:
|
|
236
|
+
raise DestinationDirectoryError(
|
|
237
|
+
"This platform cannot atomically publish a server-named download through directory descriptors."
|
|
238
|
+
)
|
|
239
|
+
os.rename(source, destination)
|
|
240
|
+
return
|
|
241
|
+
if sys.platform.startswith("linux"):
|
|
242
|
+
rename_function = _renameat2
|
|
243
|
+
flag = _RENAME_NOREPLACE
|
|
244
|
+
elif sys.platform == "darwin":
|
|
245
|
+
rename_function = _renameatx_np
|
|
246
|
+
flag = _RENAME_EXCL
|
|
247
|
+
else:
|
|
248
|
+
rename_function = None
|
|
249
|
+
flag = 0
|
|
250
|
+
if rename_function is None:
|
|
251
|
+
raise DestinationDirectoryError(
|
|
252
|
+
"This platform cannot atomically publish a server-named download without replacing an existing file."
|
|
253
|
+
)
|
|
254
|
+
old_dir_fd = _AT_FDCWD if src_dir_fd is None else src_dir_fd
|
|
255
|
+
new_dir_fd = _AT_FDCWD if dst_dir_fd is None else dst_dir_fd
|
|
256
|
+
ctypes.set_errno(0)
|
|
257
|
+
result = rename_function(
|
|
258
|
+
old_dir_fd,
|
|
259
|
+
os.fsencode(source),
|
|
260
|
+
new_dir_fd,
|
|
261
|
+
os.fsencode(destination),
|
|
262
|
+
flag,
|
|
263
|
+
)
|
|
264
|
+
if result == 0:
|
|
265
|
+
return
|
|
266
|
+
error_number = ctypes.get_errno()
|
|
267
|
+
if error_number == errno.EEXIST:
|
|
268
|
+
raise FileExistsError(error_number, os.strerror(error_number), destination)
|
|
269
|
+
if error_number in (errno.EINVAL, errno.ENOSYS, errno.EOPNOTSUPP):
|
|
270
|
+
raise DestinationDirectoryError(
|
|
271
|
+
"This filesystem cannot atomically publish a server-named download without replacing an existing file."
|
|
272
|
+
)
|
|
273
|
+
raise OSError(error_number, os.strerror(error_number), destination)
|
|
274
|
+
|
|
167
275
|
|
|
168
276
|
def _random_temp_name(final_name: str) -> str:
|
|
169
277
|
"""One candidate temp-file name for ``final_name``, unpredictable
|
|
@@ -174,6 +282,10 @@ def _random_temp_name(final_name: str) -> str:
|
|
|
174
282
|
return f".{final_name}.{secrets.token_hex(8)}.part"
|
|
175
283
|
|
|
176
284
|
|
|
285
|
+
def _portable_name_key(name: str) -> str:
|
|
286
|
+
return unicodedata.normalize("NFKC", name).casefold()
|
|
287
|
+
|
|
288
|
+
|
|
177
289
|
def _validate_leaf_name(final_name: str) -> None:
|
|
178
290
|
"""Refuse anything that is not a bare basename.
|
|
179
291
|
|
|
@@ -244,15 +356,22 @@ class DestinationDirectory:
|
|
|
244
356
|
#: re-resolved for a filesystem operation once the handle exists.
|
|
245
357
|
display_path: Path
|
|
246
358
|
|
|
247
|
-
def write_atomic(self, final_name: str, data: bytes) -> None:
|
|
359
|
+
def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
|
|
248
360
|
"""Write *data* into ``final_name`` directly inside this directory.
|
|
249
361
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
362
|
+
A same-directory temporary file is written, flushed, and ``fsync``'d,
|
|
363
|
+
then published atomically. With ``replace_existing=True`` it replaces
|
|
364
|
+
a prior complete entry; otherwise publication fails if *any* final
|
|
365
|
+
entry already exists. Any failure after temporary-file creation may
|
|
366
|
+
leave that mode-0600 ``.part`` entry behind. No backend deletes it
|
|
367
|
+
automatically because a pathname can be replaced before cleanup.
|
|
368
|
+
Ordinary write and publication errors name the temporary entry and ask
|
|
369
|
+
the operator to locate and verify it before manual cleanup.
|
|
370
|
+
``KeyboardInterrupt`` and
|
|
371
|
+
``SystemExit`` are re-raised unchanged; their possible residue is
|
|
372
|
+
documented here rather than reported while interrupting the caller.
|
|
373
|
+
``final_name`` must be a bare basename (no path separators); see
|
|
374
|
+
:func:`_validate_leaf_name`.
|
|
256
375
|
"""
|
|
257
376
|
raise NotImplementedError
|
|
258
377
|
|
|
@@ -309,26 +428,85 @@ class _DirFdDestinationDirectory(DestinationDirectory):
|
|
|
309
428
|
"after repeated attempts."
|
|
310
429
|
) from last_exc
|
|
311
430
|
|
|
312
|
-
def
|
|
431
|
+
def _refuse_portable_alias(self, final_name: str, *, published_identity: Optional[tuple[int, int]] = None) -> None:
|
|
432
|
+
aliases = [
|
|
433
|
+
name
|
|
434
|
+
for name in os.listdir(self._fd)
|
|
435
|
+
if name != final_name and _portable_name_key(name) == _portable_name_key(final_name)
|
|
436
|
+
]
|
|
437
|
+
if not aliases:
|
|
438
|
+
return
|
|
439
|
+
if published_identity is not None:
|
|
440
|
+
current = os.stat(final_name, dir_fd=self._fd, follow_symlinks=False)
|
|
441
|
+
if (current.st_dev, current.st_ino) != published_identity:
|
|
442
|
+
raise DestinationDirectoryError(
|
|
443
|
+
f"A portable-equivalent alias appeared after the complete download was published at "
|
|
444
|
+
f"{final_name!r}, and another process replaced that entry before identity verification. "
|
|
445
|
+
"Manual resolution is required; the client did not delete either entry."
|
|
446
|
+
)
|
|
447
|
+
# Python exposes no conditional unlink-by-inode primitive. Leave
|
|
448
|
+
# the proven self-published entry in place rather than reopening a
|
|
449
|
+
# stat-to-unlink race that could delete a replacement.
|
|
450
|
+
raise DestinationDirectoryError(
|
|
451
|
+
f"The complete download was published at {final_name!r}, but portable-equivalent alias "
|
|
452
|
+
f"{aliases[0]!r} appeared. Manual resolution is required; the client did not delete either entry."
|
|
453
|
+
)
|
|
454
|
+
raise DestinationDirectoryError(
|
|
455
|
+
f"Refusing to publish {final_name!r}: portable-equivalent alias {aliases[0]!r} exists."
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
|
|
313
459
|
_validate_leaf_name(final_name)
|
|
460
|
+
if not replace_existing:
|
|
461
|
+
self._refuse_portable_alias(final_name)
|
|
314
462
|
fd, temp_name = self._mkstemp(final_name)
|
|
463
|
+
try:
|
|
464
|
+
published_identity = os.fstat(fd)
|
|
465
|
+
except (KeyboardInterrupt, SystemExit):
|
|
466
|
+
_close_fd_quietly(fd)
|
|
467
|
+
raise
|
|
468
|
+
except Exception as exc:
|
|
469
|
+
_close_fd_quietly(fd)
|
|
470
|
+
raise _retained_temp_error(self.display_path, temp_name, exc) from exc
|
|
315
471
|
try:
|
|
316
472
|
handle = os.fdopen(fd, "wb")
|
|
317
|
-
except
|
|
318
|
-
|
|
473
|
+
except (KeyboardInterrupt, SystemExit):
|
|
474
|
+
_close_fd_quietly(fd)
|
|
319
475
|
raise
|
|
476
|
+
except Exception as exc:
|
|
477
|
+
_close_fd_quietly(fd)
|
|
478
|
+
raise _retained_temp_error(self.display_path, temp_name, exc) from exc
|
|
479
|
+
published = False
|
|
480
|
+
existing_final_preserved = False
|
|
320
481
|
try:
|
|
321
482
|
with handle:
|
|
322
483
|
handle.write(data)
|
|
323
484
|
handle.flush()
|
|
324
485
|
os.fsync(handle.fileno())
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
486
|
+
if replace_existing:
|
|
487
|
+
os.replace(temp_name, final_name, src_dir_fd=self._fd, dst_dir_fd=self._fd)
|
|
488
|
+
published = True
|
|
489
|
+
else:
|
|
490
|
+
try:
|
|
491
|
+
_rename_noreplace(temp_name, final_name, src_dir_fd=self._fd, dst_dir_fd=self._fd)
|
|
492
|
+
except FileExistsError as exc:
|
|
493
|
+
existing_final_preserved = True
|
|
494
|
+
raise DestinationDirectoryError(
|
|
495
|
+
f"Refusing to replace existing download destination {final_name!r} in {self.display_path}. "
|
|
496
|
+
"The existing final entry was preserved."
|
|
497
|
+
) from exc
|
|
498
|
+
published = True
|
|
499
|
+
self._refuse_portable_alias(
|
|
500
|
+
final_name, published_identity=(published_identity.st_dev, published_identity.st_ino)
|
|
501
|
+
)
|
|
502
|
+
except (KeyboardInterrupt, SystemExit):
|
|
331
503
|
raise
|
|
504
|
+
except Exception as exc:
|
|
505
|
+
if published:
|
|
506
|
+
raise
|
|
507
|
+
raise _retained_temp_error(
|
|
508
|
+
self.display_path, temp_name, exc, existing_final_preserved=existing_final_preserved
|
|
509
|
+
) from exc
|
|
332
510
|
|
|
333
511
|
def close(self) -> None:
|
|
334
512
|
if not self._closed:
|
|
@@ -398,29 +576,84 @@ class _PathIdentityDestinationDirectory(DestinationDirectory):
|
|
|
398
576
|
"after repeated attempts."
|
|
399
577
|
) from last_exc
|
|
400
578
|
|
|
401
|
-
def
|
|
579
|
+
def _refuse_portable_alias(self, final_name: str, *, published_identity: Optional[tuple[int, int]] = None) -> None:
|
|
580
|
+
aliases = [
|
|
581
|
+
path
|
|
582
|
+
for path in self.display_path.iterdir()
|
|
583
|
+
if path.name != final_name and _portable_name_key(path.name) == _portable_name_key(final_name)
|
|
584
|
+
]
|
|
585
|
+
if not aliases:
|
|
586
|
+
return
|
|
587
|
+
if published_identity is not None:
|
|
588
|
+
final_path = self.display_path / final_name
|
|
589
|
+
info = final_path.stat(follow_symlinks=False)
|
|
590
|
+
if (info.st_dev, info.st_ino) != published_identity:
|
|
591
|
+
raise DestinationDirectoryError(
|
|
592
|
+
f"A portable-equivalent alias appeared after the complete download was published at "
|
|
593
|
+
f"{final_name!r}, and another process replaced that entry before identity verification. "
|
|
594
|
+
"Manual resolution is required; the client did not delete either entry."
|
|
595
|
+
)
|
|
596
|
+
raise DestinationDirectoryError(
|
|
597
|
+
f"The complete download was published at {final_name!r}, but portable-equivalent alias "
|
|
598
|
+
f"{aliases[0].name!r} appeared. Manual resolution is required; the client did not delete either entry."
|
|
599
|
+
)
|
|
600
|
+
raise DestinationDirectoryError(f"Refusing to publish {final_name!r}: portable-equivalent alias exists.")
|
|
601
|
+
|
|
602
|
+
def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
|
|
402
603
|
_validate_leaf_name(final_name)
|
|
403
604
|
self._verify_unchanged()
|
|
605
|
+
if not replace_existing:
|
|
606
|
+
self._refuse_portable_alias(final_name)
|
|
404
607
|
fd, temp_path = self._mkstemp(final_name)
|
|
608
|
+
try:
|
|
609
|
+
published_identity = os.fstat(fd)
|
|
610
|
+
except (KeyboardInterrupt, SystemExit):
|
|
611
|
+
_close_fd_quietly(fd)
|
|
612
|
+
raise
|
|
613
|
+
except Exception as exc:
|
|
614
|
+
_close_fd_quietly(fd)
|
|
615
|
+
raise _retained_temp_error(self.display_path, temp_path.name, exc) from exc
|
|
405
616
|
final_path = self.display_path / final_name
|
|
406
617
|
try:
|
|
407
618
|
handle = os.fdopen(fd, "wb")
|
|
408
|
-
except
|
|
409
|
-
|
|
619
|
+
except (KeyboardInterrupt, SystemExit):
|
|
620
|
+
_close_fd_quietly(fd)
|
|
410
621
|
raise
|
|
622
|
+
except Exception as exc:
|
|
623
|
+
_close_fd_quietly(fd)
|
|
624
|
+
raise _retained_temp_error(self.display_path, temp_path.name, exc) from exc
|
|
625
|
+
published = False
|
|
626
|
+
existing_final_preserved = False
|
|
411
627
|
try:
|
|
412
628
|
with handle:
|
|
413
629
|
handle.write(data)
|
|
414
630
|
handle.flush()
|
|
415
631
|
os.fsync(handle.fileno())
|
|
416
632
|
self._verify_unchanged()
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
633
|
+
if replace_existing:
|
|
634
|
+
os.replace(os.fspath(temp_path), os.fspath(final_path))
|
|
635
|
+
published = True
|
|
636
|
+
else:
|
|
637
|
+
try:
|
|
638
|
+
_rename_noreplace(os.fspath(temp_path), os.fspath(final_path))
|
|
639
|
+
except FileExistsError as exc:
|
|
640
|
+
existing_final_preserved = True
|
|
641
|
+
raise DestinationDirectoryError(
|
|
642
|
+
f"Refusing to replace existing download destination {final_name!r} in {self.display_path}. "
|
|
643
|
+
"The existing final entry was preserved."
|
|
644
|
+
) from exc
|
|
645
|
+
published = True
|
|
646
|
+
self._refuse_portable_alias(
|
|
647
|
+
final_name, published_identity=(published_identity.st_dev, published_identity.st_ino)
|
|
648
|
+
)
|
|
649
|
+
except (KeyboardInterrupt, SystemExit):
|
|
423
650
|
raise
|
|
651
|
+
except Exception as exc:
|
|
652
|
+
if published:
|
|
653
|
+
raise
|
|
654
|
+
raise _retained_temp_error(
|
|
655
|
+
self.display_path, temp_path.name, exc, existing_final_preserved=existing_final_preserved
|
|
656
|
+
) from exc
|
|
424
657
|
|
|
425
658
|
def close(self) -> None:
|
|
426
659
|
self._closed = True
|
|
@@ -526,6 +759,39 @@ class DestinationObservation:
|
|
|
526
759
|
)
|
|
527
760
|
self._consumed = True
|
|
528
761
|
|
|
762
|
+
def is_existing_directory(self) -> bool:
|
|
763
|
+
"""Whether the selected directory existed when this observation began."""
|
|
764
|
+
if self.absent_names is not None:
|
|
765
|
+
return not self.absent_names
|
|
766
|
+
return self.existed
|
|
767
|
+
|
|
768
|
+
def matches_directory(self, directory: Path) -> bool:
|
|
769
|
+
"""Whether *directory* is exactly the path this observation owns."""
|
|
770
|
+
return self.expanded == Path(os.path.expanduser(os.fspath(directory)))
|
|
771
|
+
|
|
772
|
+
def contained_destination(self, final_name: str) -> Path:
|
|
773
|
+
"""Return a final path that passed this observation's containment check.
|
|
774
|
+
|
|
775
|
+
The later open consumes this same observation, so a selected directory
|
|
776
|
+
swapped after this check cannot redirect the write. A swap visible to
|
|
777
|
+
the path-resolution check instead causes a refusal.
|
|
778
|
+
"""
|
|
779
|
+
_validate_leaf_name(final_name)
|
|
780
|
+
destination = self.expanded / final_name
|
|
781
|
+
try:
|
|
782
|
+
resolved_root = self.expanded.resolve(strict=False)
|
|
783
|
+
resolved_destination = destination.resolve(strict=False)
|
|
784
|
+
resolved_destination.relative_to(resolved_root)
|
|
785
|
+
except (OSError, RuntimeError, ValueError) as exc:
|
|
786
|
+
raise DestinationDirectoryError(
|
|
787
|
+
f"Could not validate {destination} beneath selected directory {self.expanded}: {exc}"
|
|
788
|
+
) from exc
|
|
789
|
+
if resolved_destination == resolved_root:
|
|
790
|
+
raise DestinationDirectoryError(
|
|
791
|
+
f"Refusing to write {final_name!r}: it resolves to the selected directory itself."
|
|
792
|
+
)
|
|
793
|
+
return destination
|
|
794
|
+
|
|
529
795
|
def close(self) -> None:
|
|
530
796
|
"""Release the held descriptor, if any, and mark this observation
|
|
531
797
|
spent — a later :func:`open_destination_directory` call on it must
|
|
@@ -595,11 +861,20 @@ def _observe_posix(directory: Path) -> DestinationObservation:
|
|
|
595
861
|
# deliberately, not closed: this is what pins its inode against reuse
|
|
596
862
|
# for the entire gap until `open_destination_directory` runs. See the
|
|
597
863
|
# module docstring.
|
|
864
|
+
was_symlink = False
|
|
865
|
+
if not absent_names:
|
|
866
|
+
try:
|
|
867
|
+
was_symlink = stat_module.S_ISLNK(os.lstat(os.fspath(expanded)).st_mode)
|
|
868
|
+
except OSError as exc:
|
|
869
|
+
if current_fd is not None:
|
|
870
|
+
os.close(current_fd)
|
|
871
|
+
raise DestinationDirectoryError(f"Could not observe download destination {expanded}: {exc}") from exc
|
|
598
872
|
return DestinationObservation(
|
|
599
873
|
expanded,
|
|
600
874
|
is_absolute=is_absolute,
|
|
601
875
|
held_fd=current_fd,
|
|
602
876
|
absent_names=absent_names,
|
|
877
|
+
was_symlink=was_symlink,
|
|
603
878
|
)
|
|
604
879
|
|
|
605
880
|
|
|
@@ -739,6 +1014,23 @@ def _open_posix(observation: DestinationObservation) -> _DirFdDestinationDirecto
|
|
|
739
1014
|
current_fd = observation.held_fd
|
|
740
1015
|
observation.held_fd = None
|
|
741
1016
|
try:
|
|
1017
|
+
if current_fd is not None and not observation.absent_names:
|
|
1018
|
+
try:
|
|
1019
|
+
leaf_lstat = os.lstat(os.fspath(expanded))
|
|
1020
|
+
leaf_stat = os.stat(os.fspath(expanded), follow_symlinks=True)
|
|
1021
|
+
except OSError as exc:
|
|
1022
|
+
raise DestinationDirectoryError(
|
|
1023
|
+
f"Download destination {expanded} became unreachable during the download: {exc}"
|
|
1024
|
+
) from exc
|
|
1025
|
+
if not observation.was_symlink and stat_module.S_ISLNK(leaf_lstat.st_mode):
|
|
1026
|
+
raise DestinationDirectoryError(
|
|
1027
|
+
f"Refusing to write into {expanded}: it became a symlink during the download."
|
|
1028
|
+
)
|
|
1029
|
+
held_stat = os.fstat(current_fd)
|
|
1030
|
+
if (leaf_stat.st_dev, leaf_stat.st_ino) != (held_stat.st_dev, held_stat.st_ino):
|
|
1031
|
+
raise DestinationDirectoryError(
|
|
1032
|
+
f"Refusing to write into {expanded}: it was replaced with a different directory during the download."
|
|
1033
|
+
)
|
|
742
1034
|
if current_fd is None and not observation.is_absolute:
|
|
743
1035
|
# Nothing existed below the anchor, and the anchor itself is
|
|
744
1036
|
# the process's own current working directory (an absolute
|
|
@@ -45,6 +45,7 @@ from simulo.interfaces.platform.enums import ResumePolicy
|
|
|
45
45
|
|
|
46
46
|
if TYPE_CHECKING: # typing-only; avoids a module-scope dependency on the HTTP client
|
|
47
47
|
from simulo._client.preflight_api import PreflightOutcome
|
|
48
|
+
from simulo.interfaces.platform.manifest import PreflightFinding
|
|
48
49
|
|
|
49
50
|
F = TypeVar("F", bound=Callable[..., object])
|
|
50
51
|
|
|
@@ -310,6 +311,56 @@ def _spawn_local(job_name: str, package_path: Path) -> JobHandle:
|
|
|
310
311
|
return JobHandle(job_id, package_path, job_name)
|
|
311
312
|
|
|
312
313
|
|
|
314
|
+
def _skip_preflight_hint(blocking: "Sequence[PreflightFinding]") -> str:
|
|
315
|
+
"""The follow-on sentence :func:`_handle_preflight_outcome` appends after
|
|
316
|
+
its blocking-finding count, chosen from the WHOLE batch rather than one
|
|
317
|
+
finding at a time.
|
|
318
|
+
|
|
319
|
+
For most blocking codes this is the plain hint every prior release has
|
|
320
|
+
printed: the flag still bypasses the whole preflight phase, and nothing
|
|
321
|
+
here makes a finding unbypassable — ``skip_preflight_requested``'s
|
|
322
|
+
docstring rule ("a blocking finding can never become an unbypassable
|
|
323
|
+
wall") is unchanged.
|
|
324
|
+
|
|
325
|
+
``runtime_not_available`` is the one exception, and only the ADVICE
|
|
326
|
+
changes, not the flag's behavior. The generic hint's two stated reasons
|
|
327
|
+
to bypass are "preflight is wrong about your app" and "the platform is
|
|
328
|
+
unreachable" — this finding is neither: it fires only when the
|
|
329
|
+
requested runtime id is absent from an ESTABLISHED inventory (at least
|
|
330
|
+
one worker has actually reported a runtime, so the known set describes
|
|
331
|
+
the fleet rather than merely restating the default); when nothing has
|
|
332
|
+
ever reported, that same absence is the advisory ``unknown_runtime``
|
|
333
|
+
instead (see ``preflight/service.py``'s ``_runtime_availability_finding``).
|
|
334
|
+
That established-inventory fact is real regardless of workload — the
|
|
335
|
+
same trigger fires for an unavailable GPU runtime as for an unavailable
|
|
336
|
+
CPU one. Bypassing it does not route around a wrong opinion, it
|
|
337
|
+
re-enables the exact silent substitution onto the platform's GPU-backed
|
|
338
|
+
default runtime that this finding exists to prevent, so the generic
|
|
339
|
+
hint is replaced with one naming that substitution WITHOUT inventing a
|
|
340
|
+
rate, an estimate, or any claim about what the workload itself needs —
|
|
341
|
+
there is no rate card (``PreflightResponse.estimate`` is reserved and
|
|
342
|
+
always ``null``; see ``preflight/schemas.py``/``routes.py``). Keyed on
|
|
343
|
+
``finding.code`` (the server's own vocabulary — see
|
|
344
|
+
``preflight_render.py``'s module docstring), never on message text.
|
|
345
|
+
|
|
346
|
+
``--skip-preflight`` bypasses the entire preflight call, not one finding
|
|
347
|
+
in isolation (see ``skip_preflight_requested``), so if ANY finding in
|
|
348
|
+
*blocking* is ``runtime_not_available``, the qualified sentence governs
|
|
349
|
+
the whole raise even when an ordinary, safely-bypassable finding is also
|
|
350
|
+
blocking alongside it — bypassing would still re-enable that finding's
|
|
351
|
+
harm too.
|
|
352
|
+
"""
|
|
353
|
+
from simulo._client.preflight_render import RUNTIME_NOT_AVAILABLE_CODE
|
|
354
|
+
|
|
355
|
+
if any(finding.code == RUNTIME_NOT_AVAILABLE_CODE for finding in blocking):
|
|
356
|
+
return (
|
|
357
|
+
"--skip-preflight would submit anyway, but not onto the runtime you asked for — this "
|
|
358
|
+
"platform doesn't offer it, so the job would run on the platform's default runtime "
|
|
359
|
+
"instead, which is GPU-backed."
|
|
360
|
+
)
|
|
361
|
+
return "Pass --skip-preflight to submit anyway (not recommended)."
|
|
362
|
+
|
|
363
|
+
|
|
313
364
|
def _handle_preflight_outcome(outcome: "PreflightOutcome", *, job_name: str) -> None:
|
|
314
365
|
"""Print + decide for one preflight answer inside ``_spawn_cloud``.
|
|
315
366
|
|
|
@@ -331,7 +382,8 @@ def _handle_preflight_outcome(outcome: "PreflightOutcome", *, job_name: str) ->
|
|
|
331
382
|
``blocking`` finding raises :class:`PreflightBlockedError` with a short
|
|
332
383
|
stderr summary — the detailed ``✗`` lines are already on stdout by the
|
|
333
384
|
time this raises, so the exception message is a summary + the
|
|
334
|
-
``--skip-preflight`` hint, not a
|
|
385
|
+
``--skip-preflight`` hint (:func:`_skip_preflight_hint`), not a
|
|
386
|
+
duplicate render.
|
|
335
387
|
"""
|
|
336
388
|
from simulo._client.preflight_render import render_preflight_findings, render_preflight_skip_notice
|
|
337
389
|
|
|
@@ -343,7 +395,7 @@ def _handle_preflight_outcome(outcome: "PreflightOutcome", *, job_name: str) ->
|
|
|
343
395
|
return
|
|
344
396
|
raise PreflightBlockedError(
|
|
345
397
|
f"{len(blocking)} preflight issue(s) blocked job {job_name!r} — nothing was uploaded and no "
|
|
346
|
-
"job was created.
|
|
398
|
+
f"job was created. {_skip_preflight_hint(blocking)}"
|
|
347
399
|
)
|
|
348
400
|
|
|
349
401
|
|