simulo 0.31.2__tar.gz → 0.33.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {simulo-0.31.2/src/simulo.egg-info → simulo-0.33.0}/PKG-INFO +6 -6
- {simulo-0.31.2 → simulo-0.33.0}/PYPI.md +4 -4
- {simulo-0.31.2 → simulo-0.33.0}/pyproject.toml +2 -2
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/__init__.py +32 -2
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/_runner.py +7 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/_secure_downloads.py +216 -81
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/app.py +51 -9
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/asset_api.py +1 -1
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/asset_package.py +3 -3
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/bundle.py +25 -18
- simulo-0.33.0/src/simulo/_client/cli.py +317 -0
- simulo-0.33.0/src/simulo/_client/cli_arguments.py +15 -0
- simulo-0.33.0/src/simulo/_client/cli_context.py +120 -0
- simulo-0.33.0/src/simulo/_client/cli_downloads.py +387 -0
- simulo-0.33.0/src/simulo/_client/cli_errors.py +13 -0
- simulo-0.33.0/src/simulo/_client/cli_job_reporting.py +733 -0
- simulo-0.33.0/src/simulo/_client/cli_listing.py +238 -0
- simulo-0.33.0/src/simulo/_client/cli_rendering.py +187 -0
- simulo-0.33.0/src/simulo/_client/cli_text.py +392 -0
- simulo-0.33.0/src/simulo/_client/cli_waiting.py +163 -0
- simulo-0.33.0/src/simulo/_client/commands/__init__.py +1 -0
- simulo-0.33.0/src/simulo/_client/commands/assets.py +2535 -0
- simulo-0.33.0/src/simulo/_client/commands/auth.py +252 -0
- simulo-0.33.0/src/simulo/_client/commands/create.py +259 -0
- simulo-0.33.0/src/simulo/_client/commands/jobs.py +1843 -0
- simulo-0.33.0/src/simulo/_client/commands/policies.py +612 -0
- simulo-0.33.0/src/simulo/_client/commands/recordings.py +429 -0
- simulo-0.33.0/src/simulo/_client/commands/run.py +1363 -0
- simulo-0.33.0/src/simulo/_client/commands/systems.py +132 -0
- simulo-0.33.0/src/simulo/_client/evaluation_report.py +618 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/facades.py +70 -25
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/http.py +5 -2
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/jobs_api.py +43 -17
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/learning.py +76 -11
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/packaging.py +72 -37
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/policies_api.py +12 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/preflight_api.py +1 -4
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/preflight_render.py +4 -4
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/preview_report.py +64 -22
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/runtime_display.py +1 -1
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/submit_api.py +77 -21
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/__init__.py +3 -3
- simulo-0.33.0/src/simulo/_client/templates/training/eval.py.tmpl +34 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/training/task.py.tmpl +25 -17
- {simulo-0.31.2 → simulo-0.33.0/src/simulo.egg-info}/PKG-INFO +6 -6
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo.egg-info/SOURCES.txt +20 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo.egg-info/requires.txt +1 -1
- simulo-0.31.2/src/simulo/_client/cli.py +0 -8923
- {simulo-0.31.2 → simulo-0.33.0}/MANIFEST.in +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/setup.cfg +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/__init__.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/_arguments.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/_mounts.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/asset.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/asset_pins.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/builtin_aliases.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/cancel_api.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/config.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/credentials.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/discovery.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/export_api.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/identity_api.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/install_samples.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/login.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/mode.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/onnx_bundle.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/outputs.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/registry.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/runtime.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/stub.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/training/play.py.tmpl +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/training/preview.py.tmpl +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/training/simuloignore.tmpl +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/training/train.py.tmpl +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/view_fragment.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/view_session_api.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/volume.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/callbacks.py +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo/py.typed +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo.egg-info/dependency_links.txt +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo.egg-info/entry_points.txt +0 -0
- {simulo-0.31.2 → simulo-0.33.0}/src/simulo.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: simulo
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.33.0
|
|
4
4
|
Summary: Python SDK and CLI for cloud-native robotics simulation, reinforcement learning, and robot policy training on managed GPUs.
|
|
5
5
|
Author-email: Simulo Team <team@simulo.ai>
|
|
6
6
|
License: BSD-3-Clause
|
|
@@ -31,7 +31,7 @@ Classifier: Topic :: System :: Distributed Computing
|
|
|
31
31
|
Classifier: Typing :: Typed
|
|
32
32
|
Requires-Python: >=3.11
|
|
33
33
|
Description-Content-Type: text/markdown
|
|
34
|
-
Requires-Dist: simulo-interfaces<0.
|
|
34
|
+
Requires-Dist: simulo-interfaces<0.25,>=0.24.0
|
|
35
35
|
Requires-Dist: mcap<2,>=1.3
|
|
36
36
|
Requires-Dist: defusedxml>=0.7.1
|
|
37
37
|
Requires-Dist: usd-core<27,>=26.5
|
|
@@ -94,7 +94,7 @@ simulo login
|
|
|
94
94
|
cd mybot
|
|
95
95
|
simulo run preview.py --frozen --strict-assets
|
|
96
96
|
simulo run train.py --frozen --strict-assets
|
|
97
|
-
simulo run play.py --policy
|
|
97
|
+
simulo run play.py --policy <policy-id>:best --frozen --strict-assets
|
|
98
98
|
```
|
|
99
99
|
|
|
100
100
|
`simulo create` writes a complete starter application to your machine without
|
|
@@ -115,7 +115,7 @@ See the repository README for setup and usage.
|
|
|
115
115
|
|
|
116
116
|
```bash
|
|
117
117
|
simulo jobs # list submitted jobs and status
|
|
118
|
-
simulo logs --follow # stream
|
|
118
|
+
simulo logs --follow # stream logs from your most recent job you submitted (train, play, preview or eval)
|
|
119
119
|
simulo result # read the job's result
|
|
120
120
|
simulo policy list # list training policies and checkpoints
|
|
121
121
|
simulo policy get policy_x:best # download a verified checkpoint
|
|
@@ -126,8 +126,8 @@ simulo view # view a running --viewstream job in 3D
|
|
|
126
126
|
simulo cancel # stop a queued or running job
|
|
127
127
|
```
|
|
128
128
|
|
|
129
|
-
Commands that accept a job ID default to the latest
|
|
130
|
-
|
|
129
|
+
Commands that accept a job ID default to the latest user-submitted job
|
|
130
|
+
(`train`, `play`, `preview`, or `eval`) and tell you its ID and type.
|
|
131
131
|
|
|
132
132
|
## Use versioned robot assets
|
|
133
133
|
|
|
@@ -38,7 +38,7 @@ simulo login
|
|
|
38
38
|
cd mybot
|
|
39
39
|
simulo run preview.py --frozen --strict-assets
|
|
40
40
|
simulo run train.py --frozen --strict-assets
|
|
41
|
-
simulo run play.py --policy
|
|
41
|
+
simulo run play.py --policy <policy-id>:best --frozen --strict-assets
|
|
42
42
|
```
|
|
43
43
|
|
|
44
44
|
`simulo create` writes a complete starter application to your machine without
|
|
@@ -59,7 +59,7 @@ See the repository README for setup and usage.
|
|
|
59
59
|
|
|
60
60
|
```bash
|
|
61
61
|
simulo jobs # list submitted jobs and status
|
|
62
|
-
simulo logs --follow # stream
|
|
62
|
+
simulo logs --follow # stream logs from your most recent job you submitted (train, play, preview or eval)
|
|
63
63
|
simulo result # read the job's result
|
|
64
64
|
simulo policy list # list training policies and checkpoints
|
|
65
65
|
simulo policy get policy_x:best # download a verified checkpoint
|
|
@@ -70,8 +70,8 @@ simulo view # view a running --viewstream job in 3D
|
|
|
70
70
|
simulo cancel # stop a queued or running job
|
|
71
71
|
```
|
|
72
72
|
|
|
73
|
-
Commands that accept a job ID default to the latest
|
|
74
|
-
|
|
73
|
+
Commands that accept a job ID default to the latest user-submitted job
|
|
74
|
+
(`train`, `play`, `preview`, or `eval`) and tell you its ID and type.
|
|
75
75
|
|
|
76
76
|
## Use versioned robot assets
|
|
77
77
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "simulo"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.33.0"
|
|
8
8
|
description = "Python SDK and CLI for cloud-native robotics simulation, reinforcement learning, and robot policy training on managed GPUs."
|
|
9
9
|
readme = "PYPI.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -59,7 +59,7 @@ classifiers = [
|
|
|
59
59
|
# mcap: used by `simulo recordings` to read back a downloaded recording and
|
|
60
60
|
# verify it (message count) — MIT-licensed, deps only lz4+zstandard.
|
|
61
61
|
dependencies = [
|
|
62
|
-
"simulo-interfaces>=0.
|
|
62
|
+
"simulo-interfaces>=0.24.0,<0.25",
|
|
63
63
|
"mcap>=1.3,<2",
|
|
64
64
|
# Hardened XML parsing for URDF ingestion (USD Asset Catalogs, PR-12 fix
|
|
65
65
|
# loop 2, security NIT): stdlib xml.etree.ElementTree relies on
|
|
@@ -59,7 +59,9 @@ Three kinds of name live on ``simulo``:
|
|
|
59
59
|
prebuilt place you bring, ``World.usd(...)``.)
|
|
60
60
|
"""
|
|
61
61
|
|
|
62
|
+
import dis as _dis
|
|
62
63
|
import pkgutil as _pkgutil
|
|
64
|
+
import sys as _sys
|
|
63
65
|
from typing import TYPE_CHECKING
|
|
64
66
|
|
|
65
67
|
__path__ = _pkgutil.extend_path(__path__, __name__)
|
|
@@ -143,6 +145,8 @@ from simulo.interfaces.authoring import (
|
|
|
143
145
|
LightConfig,
|
|
144
146
|
PointMarker,
|
|
145
147
|
PolicyCheckpoint,
|
|
148
|
+
PolicyInput,
|
|
149
|
+
PolicyInputSchema,
|
|
146
150
|
PrimitiveShape,
|
|
147
151
|
RigidPhysics,
|
|
148
152
|
RoughTerrain,
|
|
@@ -188,6 +192,8 @@ __all__ = [
|
|
|
188
192
|
"JobPublicId",
|
|
189
193
|
"PolicyPublicId",
|
|
190
194
|
"PolicyNetworkProtocol",
|
|
195
|
+
"PolicyInput",
|
|
196
|
+
"PolicyInputSchema",
|
|
191
197
|
"RecordingPublicId",
|
|
192
198
|
"OutputPublicId",
|
|
193
199
|
"parse_job_public_id",
|
|
@@ -234,6 +240,8 @@ __all__ = [
|
|
|
234
240
|
"PointMarker",
|
|
235
241
|
# lazy, mode-aware (the learning names + save_output)
|
|
236
242
|
"ActuatorGainsConfig",
|
|
243
|
+
"ArticulationConfig",
|
|
244
|
+
"JointConfig",
|
|
237
245
|
"Camera",
|
|
238
246
|
"CameraSpawnConfig",
|
|
239
247
|
"ContactSensor",
|
|
@@ -277,11 +285,25 @@ __all__ = [
|
|
|
277
285
|
"Trainer",
|
|
278
286
|
"Visual",
|
|
279
287
|
"World",
|
|
288
|
+
"evaluate",
|
|
280
289
|
"preview",
|
|
281
290
|
"save_output",
|
|
282
291
|
]
|
|
283
292
|
|
|
284
293
|
|
|
294
|
+
def _is_from_import_lookup() -> bool:
|
|
295
|
+
"""True when the running lookup is the ``IMPORT_FROM`` of ``from simulo import X``.
|
|
296
|
+
|
|
297
|
+
Attribute access and ``hasattr`` keep raising ``AttributeError``; only a ``from``
|
|
298
|
+
import needs the ``ImportError`` that carries the message. Any doubt answers False.
|
|
299
|
+
"""
|
|
300
|
+
try:
|
|
301
|
+
caller = _sys._getframe(2) # skip this helper and ``__getattr__``
|
|
302
|
+
return _dis.opname[caller.f_code.co_code[caller.f_lasti]] == "IMPORT_FROM"
|
|
303
|
+
except (ValueError, IndexError):
|
|
304
|
+
return False
|
|
305
|
+
|
|
306
|
+
|
|
285
307
|
def __getattr__(name: str) -> object:
|
|
286
308
|
"""PEP 562 hook: lazily resolve the mode-aware learning names.
|
|
287
309
|
|
|
@@ -298,6 +320,10 @@ def __getattr__(name: str) -> object:
|
|
|
298
320
|
"run": "simulo.run was removed. Use `simulo run train.py` instead.",
|
|
299
321
|
}
|
|
300
322
|
if name in removed_hints:
|
|
323
|
+
if _is_from_import_lookup():
|
|
324
|
+
# CPython swallows an AttributeError raised during ``from simulo import X``
|
|
325
|
+
# and reports a bare "cannot import name", so the pointer would be lost.
|
|
326
|
+
raise ImportError(removed_hints[name], name=__name__)
|
|
301
327
|
raise AttributeError(removed_hints[name])
|
|
302
328
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
303
329
|
|
|
@@ -350,11 +376,11 @@ if TYPE_CHECKING:
|
|
|
350
376
|
# ``simulo.interfaces.authoring`` (``CameraProtocol`` / ``PropProtocol``)
|
|
351
377
|
# and both are instantiated, never subclassed (``simulo.Camera(width=640,
|
|
352
378
|
# ...)``, ``simulo.Prop(shape)``), so each follows the "instantiated mirror
|
|
353
|
-
# -> facade" rule above exactly. The other
|
|
379
|
+
# -> facade" rule above exactly. The other names
|
|
354
380
|
# have no TYPE_CHECKING alias, DELIBERATELY, per the rule's own "where none
|
|
355
381
|
# exists, do not invent one" corollary:
|
|
356
382
|
# - ``CameraSpawnConfig`` / ``SensorOffset`` / ``RayPattern`` /
|
|
357
|
-
# ``
|
|
383
|
+
# ``GripperState`` / ``GripperCommand`` /
|
|
358
384
|
# ``Simulation`` have no contract Protocol at all — nothing to alias
|
|
359
385
|
# against.
|
|
360
386
|
# - ``ContactSensor`` / ``RayCaster`` / ``IMU`` / ``ForceTorqueSensor`` /
|
|
@@ -381,6 +407,7 @@ if TYPE_CHECKING:
|
|
|
381
407
|
from simulo._client.facades import RLPlayer as RLPlayer
|
|
382
408
|
from simulo._client.facades import RLTrainer as RLTrainer
|
|
383
409
|
from simulo._client.facades import Robot as Robot
|
|
410
|
+
from simulo._client.facades import evaluate as evaluate
|
|
384
411
|
from simulo._client.facades import preview as preview
|
|
385
412
|
from simulo.interfaces.authoring import SceneProtocol
|
|
386
413
|
from simulo.interfaces.runtime import (
|
|
@@ -414,7 +441,10 @@ if TYPE_CHECKING:
|
|
|
414
441
|
# ``simulo.interfaces.authoring`` class in both
|
|
415
442
|
# modes, so these alias the real class — precise field/signature checking
|
|
416
443
|
# (e.g. mypy rejects ``simulo.Pose(xyz="nope")``) instead of ``Any``.
|
|
444
|
+
from simulo.interfaces.authoring import ActuatorGainsConfig as ActuatorGainsConfig
|
|
445
|
+
from simulo.interfaces.authoring import ArticulationConfig as ArticulationConfig
|
|
417
446
|
from simulo.interfaces.authoring import Entity as Entity
|
|
447
|
+
from simulo.interfaces.authoring import JointConfig as JointConfig
|
|
418
448
|
from simulo.interfaces.authoring import Light as Light
|
|
419
449
|
from simulo.interfaces.authoring import Material as Material
|
|
420
450
|
from simulo.interfaces.authoring import Physics as Physics
|
|
@@ -92,6 +92,7 @@ class JobHandle:
|
|
|
92
92
|
token: Optional[str] = None,
|
|
93
93
|
job_type: Optional[JobType] = None,
|
|
94
94
|
input_checkpoint: Optional[str] = None,
|
|
95
|
+
compare_checkpoint: Optional[str] = None,
|
|
95
96
|
deferred_stdout: str = "",
|
|
96
97
|
deferred_stderr: str = "",
|
|
97
98
|
) -> None:
|
|
@@ -104,6 +105,7 @@ class JobHandle:
|
|
|
104
105
|
self._token = token
|
|
105
106
|
self._job_type = job_type
|
|
106
107
|
self._input_checkpoint = input_checkpoint
|
|
108
|
+
self._compare_checkpoint = compare_checkpoint
|
|
107
109
|
self._deferred_stdout = deferred_stdout
|
|
108
110
|
self._deferred_stderr = deferred_stderr
|
|
109
111
|
|
|
@@ -152,6 +154,11 @@ class JobHandle:
|
|
|
152
154
|
"""Policy checkpoint requested with ``simulo run --from``."""
|
|
153
155
|
return self._input_checkpoint
|
|
154
156
|
|
|
157
|
+
@property
|
|
158
|
+
def compare_checkpoint(self) -> Optional[str]:
|
|
159
|
+
"""Policy checkpoint requested with ``simulo run --compare``."""
|
|
160
|
+
return self._compare_checkpoint
|
|
161
|
+
|
|
155
162
|
@property
|
|
156
163
|
def deferred_stdout(self) -> str:
|
|
157
164
|
"""Submit-time output held until the cloud job header is printed."""
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
THE RACE THIS CLOSES. The bulk download path validates its assembled
|
|
4
4
|
filename and the selected destination directory as PATH STRINGS
|
|
5
|
-
(``_contained_bulk_download_destination`` in ``
|
|
5
|
+
(``_contained_bulk_download_destination`` in ``cli_downloads.py``: resolve, then
|
|
6
6
|
``relative_to``) before any bytes are fetched. The actual write — inside
|
|
7
7
|
this client's checkpoint/output/recording download paths
|
|
8
8
|
— re-derives the SAME destination from its path string again, after a
|
|
@@ -112,6 +112,7 @@ from __future__ import annotations
|
|
|
112
112
|
|
|
113
113
|
import ctypes
|
|
114
114
|
import errno
|
|
115
|
+
import logging
|
|
115
116
|
import os
|
|
116
117
|
import secrets
|
|
117
118
|
import stat as stat_module
|
|
@@ -136,7 +137,8 @@ class DestinationDirectoryError(OSError):
|
|
|
136
137
|
|
|
137
138
|
An ``OSError`` subclass so every existing ``except (OSError, ValueError)``
|
|
138
139
|
around a download write (``_download_and_report`` and its Model/Output
|
|
139
|
-
counterparts in ``
|
|
140
|
+
counterparts in ``commands/jobs.py``, ``commands/policies.py``, and
|
|
141
|
+
``commands/recordings.py``) already reports it as one clean CLI line;
|
|
140
142
|
no caller needs a new except clause to handle this module's refusals.
|
|
141
143
|
"""
|
|
142
144
|
|
|
@@ -155,6 +157,38 @@ def _retained_temp_error(
|
|
|
155
157
|
)
|
|
156
158
|
|
|
157
159
|
|
|
160
|
+
def _write_failure_error(
|
|
161
|
+
destination_display_path: Path,
|
|
162
|
+
final_name: str,
|
|
163
|
+
temp_name: str,
|
|
164
|
+
exc: Exception,
|
|
165
|
+
*,
|
|
166
|
+
discarded: bool,
|
|
167
|
+
existing_final_preserved: bool = False,
|
|
168
|
+
) -> DestinationDirectoryError:
|
|
169
|
+
"""The error for a write that failed before publication.
|
|
170
|
+
|
|
171
|
+
When the temporary entry was removed the user needs only the reason; the
|
|
172
|
+
retained-entry wording is for the case where it could not be removed.
|
|
173
|
+
"""
|
|
174
|
+
if not discarded:
|
|
175
|
+
return _retained_temp_error(
|
|
176
|
+
destination_display_path, temp_name, exc, existing_final_preserved=existing_final_preserved
|
|
177
|
+
)
|
|
178
|
+
if isinstance(exc, DestinationDirectoryError):
|
|
179
|
+
return exc
|
|
180
|
+
return DestinationDirectoryError(
|
|
181
|
+
f"Could not write {final_name!r} into {destination_display_path}: {exc}. " "The partial download was removed."
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _existing_final_message(final_name: str, destination_display_path: Path) -> str:
|
|
186
|
+
return (
|
|
187
|
+
f"{final_name!r} already exists in {destination_display_path}; "
|
|
188
|
+
"remove it, or download it to another folder with -o DIR."
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
158
192
|
def _close_fd_quietly(fd: int) -> None:
|
|
159
193
|
try:
|
|
160
194
|
os.close(fd)
|
|
@@ -194,8 +228,8 @@ _TEMP_NAME_ATTEMPTS = 64
|
|
|
194
228
|
# or an existing final entry is left untouched. ``link`` followed by
|
|
195
229
|
# ``unlink`` cannot offer that postcondition because a process death between
|
|
196
230
|
# those calls leaves both names behind. Keep the optional libc binding at
|
|
197
|
-
# module scope
|
|
198
|
-
#
|
|
231
|
+
# module scope; a libc, platform or filesystem without the primitive falls back
|
|
232
|
+
# to :func:`_publish_by_link`, which still never replaces an existing file.
|
|
199
233
|
_AT_FDCWD = -100
|
|
200
234
|
_RENAME_NOREPLACE = 1
|
|
201
235
|
_RENAME_EXCL = 0x00000004
|
|
@@ -221,14 +255,16 @@ def _rename_noreplace(
|
|
|
221
255
|
*,
|
|
222
256
|
src_dir_fd: Optional[int] = None,
|
|
223
257
|
dst_dir_fd: Optional[int] = None,
|
|
224
|
-
) ->
|
|
258
|
+
) -> bool:
|
|
225
259
|
"""Atomically publish *source* at absent *destination*, or fail closed.
|
|
226
260
|
|
|
227
261
|
Linux uses ``renameat2(RENAME_NOREPLACE)`` and macOS uses
|
|
228
262
|
``renameatx_np(RENAME_EXCL)``. Windows' native ``os.rename`` refuses an
|
|
229
|
-
existing destination.
|
|
230
|
-
|
|
231
|
-
|
|
263
|
+
existing destination. Where the platform, or the filesystem under the
|
|
264
|
+
destination (an NTFS or FAT mount), has no such primitive,
|
|
265
|
+
:func:`_publish_by_link` publishes without ever replacing an existing
|
|
266
|
+
file. Return ``True`` when linking leaves the temporary name for the caller
|
|
267
|
+
to discard after recording publication. Explicit user file targets keep
|
|
232
268
|
using ``os.replace`` through ``replace_existing=True``.
|
|
233
269
|
"""
|
|
234
270
|
if os.name == "nt":
|
|
@@ -237,7 +273,7 @@ def _rename_noreplace(
|
|
|
237
273
|
"This platform cannot atomically publish a server-named download through directory descriptors."
|
|
238
274
|
)
|
|
239
275
|
os.rename(source, destination)
|
|
240
|
-
return
|
|
276
|
+
return False
|
|
241
277
|
if sys.platform.startswith("linux"):
|
|
242
278
|
rename_function = _renameat2
|
|
243
279
|
flag = _RENAME_NOREPLACE
|
|
@@ -248,9 +284,8 @@ def _rename_noreplace(
|
|
|
248
284
|
rename_function = None
|
|
249
285
|
flag = 0
|
|
250
286
|
if rename_function is None:
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
)
|
|
287
|
+
_publish_by_link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd)
|
|
288
|
+
return True
|
|
254
289
|
old_dir_fd = _AT_FDCWD if src_dir_fd is None else src_dir_fd
|
|
255
290
|
new_dir_fd = _AT_FDCWD if dst_dir_fd is None else dst_dir_fd
|
|
256
291
|
ctypes.set_errno(0)
|
|
@@ -262,17 +297,43 @@ def _rename_noreplace(
|
|
|
262
297
|
flag,
|
|
263
298
|
)
|
|
264
299
|
if result == 0:
|
|
265
|
-
return
|
|
300
|
+
return False
|
|
266
301
|
error_number = ctypes.get_errno()
|
|
267
302
|
if error_number == errno.EEXIST:
|
|
268
303
|
raise FileExistsError(error_number, os.strerror(error_number), destination)
|
|
269
|
-
if error_number in (errno.EINVAL, errno.ENOSYS, errno.EOPNOTSUPP):
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
)
|
|
304
|
+
if error_number in (errno.EINVAL, errno.ENOSYS, errno.EOPNOTSUPP, errno.ENOTSUP):
|
|
305
|
+
# The filesystem (an NTFS or FAT mount, a network share) does not
|
|
306
|
+
# implement the no-replace flag.
|
|
307
|
+
_publish_by_link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd)
|
|
308
|
+
return True
|
|
273
309
|
raise OSError(error_number, os.strerror(error_number), destination)
|
|
274
310
|
|
|
275
311
|
|
|
312
|
+
def _publish_by_link(
|
|
313
|
+
source: str,
|
|
314
|
+
destination: str,
|
|
315
|
+
*,
|
|
316
|
+
src_dir_fd: Optional[int] = None,
|
|
317
|
+
dst_dir_fd: Optional[int] = None,
|
|
318
|
+
) -> None:
|
|
319
|
+
"""Publish with an atomic no-clobber hard link.
|
|
320
|
+
|
|
321
|
+
Unlike an exclusive-create claim followed by replacement, ``link`` refuses
|
|
322
|
+
any occupied destination at publication time. The caller discards the
|
|
323
|
+
temporary name after recording publication, checking its identity first.
|
|
324
|
+
Filesystems without hard links are refused rather than risking replacement.
|
|
325
|
+
"""
|
|
326
|
+
try:
|
|
327
|
+
os.link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd, follow_symlinks=False)
|
|
328
|
+
except FileExistsError:
|
|
329
|
+
raise
|
|
330
|
+
except (OSError, NotImplementedError) as exc:
|
|
331
|
+
raise DestinationDirectoryError(
|
|
332
|
+
"This filesystem cannot safely publish a download without replacing an existing file: "
|
|
333
|
+
f"the atomic hard link failed ({exc}). Choose another download folder with -o DIR."
|
|
334
|
+
) from exc
|
|
335
|
+
|
|
336
|
+
|
|
276
337
|
def _random_temp_name(final_name: str) -> str:
|
|
277
338
|
"""One candidate temp-file name for ``final_name``, unpredictable
|
|
278
339
|
enough that an attacker cannot pre-plant a symlink at it — the
|
|
@@ -362,19 +423,62 @@ class DestinationDirectory:
|
|
|
362
423
|
A same-directory temporary file is written, flushed, and ``fsync``'d,
|
|
363
424
|
then published atomically. With ``replace_existing=True`` it replaces
|
|
364
425
|
a prior complete entry; otherwise publication fails if *any* final
|
|
365
|
-
entry already exists.
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
426
|
+
entry already exists. After any failure before publication the
|
|
427
|
+
temporary entry is removed, but only when the name still refers to the
|
|
428
|
+
very file this call created (same device and inode, and for the
|
|
429
|
+
path-based writer the same directory): a pathname can be replaced
|
|
430
|
+
before cleanup, and an entry that is no longer provably ours is left
|
|
431
|
+
alone. Then the error says plainly why the write failed. Only when
|
|
432
|
+
that entry cannot be removed does the error name the retained
|
|
433
|
+
mode-0600 ``.part`` entry and ask the operator to locate and verify it
|
|
434
|
+
before manual cleanup. ``KeyboardInterrupt`` and ``SystemExit``
|
|
435
|
+
remove the entry the same way and are re-raised unchanged.
|
|
436
|
+
When publication uses a hard link, the completed temporary name is
|
|
437
|
+
removed using the same identity check; failed cleanup is reported.
|
|
373
438
|
``final_name`` must be a bare basename (no path separators); see
|
|
374
439
|
:func:`_validate_leaf_name`.
|
|
375
440
|
"""
|
|
376
441
|
raise NotImplementedError
|
|
377
442
|
|
|
443
|
+
def _discard_temp(self, temp_name: str, identity: Optional[tuple[int, int]]) -> bool:
|
|
444
|
+
raise NotImplementedError
|
|
445
|
+
|
|
446
|
+
def _discard_linked_temp(self, final_name: str, temp_name: str, identity: tuple[int, int]) -> None:
|
|
447
|
+
if not self._discard_temp(temp_name, identity):
|
|
448
|
+
logging.getLogger(__name__).warning(
|
|
449
|
+
"Download %r was published in %s. Temporary entry %r could not be safely removed. "
|
|
450
|
+
"Verify its identity before manual cleanup.",
|
|
451
|
+
final_name,
|
|
452
|
+
self.display_path,
|
|
453
|
+
temp_name,
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
def _unpublished_failure(
|
|
457
|
+
self,
|
|
458
|
+
final_name: str,
|
|
459
|
+
temp_name: str,
|
|
460
|
+
identity: Optional[tuple[int, int]],
|
|
461
|
+
exc: BaseException,
|
|
462
|
+
existing_final_preserved: bool,
|
|
463
|
+
) -> Optional[DestinationDirectoryError]:
|
|
464
|
+
"""Remove the unpublished temp entry; return the error to raise instead of *exc*.
|
|
465
|
+
|
|
466
|
+
``None`` means re-raise *exc* unchanged: an interrupt or exit, or an
|
|
467
|
+
error that already says what it needs to.
|
|
468
|
+
"""
|
|
469
|
+
discarded = self._discard_temp(temp_name, identity)
|
|
470
|
+
if not isinstance(exc, Exception):
|
|
471
|
+
return None
|
|
472
|
+
error = _write_failure_error(
|
|
473
|
+
self.display_path,
|
|
474
|
+
final_name,
|
|
475
|
+
temp_name,
|
|
476
|
+
exc,
|
|
477
|
+
discarded=discarded,
|
|
478
|
+
existing_final_preserved=existing_final_preserved,
|
|
479
|
+
)
|
|
480
|
+
return None if error is exc else error
|
|
481
|
+
|
|
378
482
|
def close(self) -> None:
|
|
379
483
|
raise NotImplementedError
|
|
380
484
|
|
|
@@ -455,29 +559,45 @@ class _DirFdDestinationDirectory(DestinationDirectory):
|
|
|
455
559
|
f"Refusing to publish {final_name!r}: portable-equivalent alias {aliases[0]!r} exists."
|
|
456
560
|
)
|
|
457
561
|
|
|
562
|
+
def _discard_temp(self, temp_name: str, identity: Optional[tuple[int, int]]) -> bool:
|
|
563
|
+
"""Remove the temporary entry this call created; ``True`` when it is gone.
|
|
564
|
+
|
|
565
|
+
The held directory descriptor pins the directory, so only the entry's
|
|
566
|
+
name can have changed. It is removed only while it still has the
|
|
567
|
+
identity of the file this call created. A name that now refers to
|
|
568
|
+
anything else is left alone.
|
|
569
|
+
"""
|
|
570
|
+
try:
|
|
571
|
+
if identity is None:
|
|
572
|
+
return False
|
|
573
|
+
current = os.stat(temp_name, dir_fd=self._fd, follow_symlinks=False)
|
|
574
|
+
if (current.st_dev, current.st_ino) != identity:
|
|
575
|
+
return False
|
|
576
|
+
os.unlink(temp_name, dir_fd=self._fd)
|
|
577
|
+
except FileNotFoundError:
|
|
578
|
+
return True
|
|
579
|
+
except OSError:
|
|
580
|
+
return False
|
|
581
|
+
return True
|
|
582
|
+
|
|
458
583
|
def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
|
|
459
584
|
_validate_leaf_name(final_name)
|
|
460
585
|
if not replace_existing:
|
|
461
586
|
self._refuse_portable_alias(final_name)
|
|
462
587
|
fd, temp_name = self._mkstemp(final_name)
|
|
588
|
+
identity: Optional[tuple[int, int]] = None
|
|
589
|
+
published = False
|
|
590
|
+
existing_final_preserved = False
|
|
463
591
|
try:
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
_close_fd_quietly(fd)
|
|
467
|
-
raise
|
|
468
|
-
except Exception as exc:
|
|
469
|
-
_close_fd_quietly(fd)
|
|
470
|
-
raise _retained_temp_error(self.display_path, temp_name, exc) from exc
|
|
471
|
-
try:
|
|
592
|
+
created = os.fstat(fd)
|
|
593
|
+
identity = (created.st_dev, created.st_ino)
|
|
472
594
|
handle = os.fdopen(fd, "wb")
|
|
473
|
-
except
|
|
595
|
+
except BaseException as exc:
|
|
474
596
|
_close_fd_quietly(fd)
|
|
597
|
+
replacement = self._unpublished_failure(final_name, temp_name, identity, exc, existing_final_preserved)
|
|
598
|
+
if replacement is not None:
|
|
599
|
+
raise replacement from exc
|
|
475
600
|
raise
|
|
476
|
-
except Exception as exc:
|
|
477
|
-
_close_fd_quietly(fd)
|
|
478
|
-
raise _retained_temp_error(self.display_path, temp_name, exc) from exc
|
|
479
|
-
published = False
|
|
480
|
-
existing_final_preserved = False
|
|
481
601
|
try:
|
|
482
602
|
with handle:
|
|
483
603
|
handle.write(data)
|
|
@@ -488,25 +608,24 @@ class _DirFdDestinationDirectory(DestinationDirectory):
|
|
|
488
608
|
published = True
|
|
489
609
|
else:
|
|
490
610
|
try:
|
|
491
|
-
_rename_noreplace(temp_name, final_name, src_dir_fd=self._fd, dst_dir_fd=self._fd)
|
|
611
|
+
linked = _rename_noreplace(temp_name, final_name, src_dir_fd=self._fd, dst_dir_fd=self._fd)
|
|
492
612
|
except FileExistsError as exc:
|
|
493
613
|
existing_final_preserved = True
|
|
494
|
-
raise DestinationDirectoryError(
|
|
495
|
-
f"Refusing to replace existing download destination {final_name!r} in {self.display_path}. "
|
|
496
|
-
"The existing final entry was preserved."
|
|
497
|
-
) from exc
|
|
614
|
+
raise DestinationDirectoryError(_existing_final_message(final_name, self.display_path)) from exc
|
|
498
615
|
published = True
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
616
|
+
assert identity is not None
|
|
617
|
+
try:
|
|
618
|
+
self._refuse_portable_alias(final_name, published_identity=identity)
|
|
619
|
+
finally:
|
|
620
|
+
if linked:
|
|
621
|
+
self._discard_linked_temp(final_name, temp_name, identity)
|
|
622
|
+
except BaseException as exc:
|
|
505
623
|
if published:
|
|
506
624
|
raise
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
625
|
+
replacement = self._unpublished_failure(final_name, temp_name, identity, exc, existing_final_preserved)
|
|
626
|
+
if replacement is not None:
|
|
627
|
+
raise replacement from exc
|
|
628
|
+
raise
|
|
510
629
|
|
|
511
630
|
def close(self) -> None:
|
|
512
631
|
if not self._closed:
|
|
@@ -599,31 +718,48 @@ class _PathIdentityDestinationDirectory(DestinationDirectory):
|
|
|
599
718
|
)
|
|
600
719
|
raise DestinationDirectoryError(f"Refusing to publish {final_name!r}: portable-equivalent alias exists.")
|
|
601
720
|
|
|
721
|
+
def _discard_temp(self, temp_name: str, identity: Optional[tuple[int, int]]) -> bool:
|
|
722
|
+
"""Remove the temporary entry this call created; ``True`` when it is gone.
|
|
723
|
+
|
|
724
|
+
Nothing holds this directory, so the directory is re-verified first and
|
|
725
|
+
the entry is removed only while it still has the identity of the file
|
|
726
|
+
this call created. Anything else is left alone.
|
|
727
|
+
"""
|
|
728
|
+
try:
|
|
729
|
+
if identity is None:
|
|
730
|
+
return False
|
|
731
|
+
self._verify_unchanged()
|
|
732
|
+
temp_path = os.fspath(self.display_path / temp_name)
|
|
733
|
+
current = os.stat(temp_path, follow_symlinks=False)
|
|
734
|
+
if (current.st_dev, current.st_ino) != identity:
|
|
735
|
+
return False
|
|
736
|
+
os.unlink(temp_path)
|
|
737
|
+
except FileNotFoundError:
|
|
738
|
+
return True
|
|
739
|
+
except OSError:
|
|
740
|
+
return False
|
|
741
|
+
return True
|
|
742
|
+
|
|
602
743
|
def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
|
|
603
744
|
_validate_leaf_name(final_name)
|
|
604
745
|
self._verify_unchanged()
|
|
605
746
|
if not replace_existing:
|
|
606
747
|
self._refuse_portable_alias(final_name)
|
|
607
748
|
fd, temp_path = self._mkstemp(final_name)
|
|
608
|
-
|
|
609
|
-
published_identity = os.fstat(fd)
|
|
610
|
-
except (KeyboardInterrupt, SystemExit):
|
|
611
|
-
_close_fd_quietly(fd)
|
|
612
|
-
raise
|
|
613
|
-
except Exception as exc:
|
|
614
|
-
_close_fd_quietly(fd)
|
|
615
|
-
raise _retained_temp_error(self.display_path, temp_path.name, exc) from exc
|
|
749
|
+
identity: Optional[tuple[int, int]] = None
|
|
616
750
|
final_path = self.display_path / final_name
|
|
751
|
+
published = False
|
|
752
|
+
existing_final_preserved = False
|
|
617
753
|
try:
|
|
754
|
+
created = os.fstat(fd)
|
|
755
|
+
identity = (created.st_dev, created.st_ino)
|
|
618
756
|
handle = os.fdopen(fd, "wb")
|
|
619
|
-
except
|
|
757
|
+
except BaseException as exc:
|
|
620
758
|
_close_fd_quietly(fd)
|
|
759
|
+
replacement = self._unpublished_failure(final_name, temp_path.name, identity, exc, existing_final_preserved)
|
|
760
|
+
if replacement is not None:
|
|
761
|
+
raise replacement from exc
|
|
621
762
|
raise
|
|
622
|
-
except Exception as exc:
|
|
623
|
-
_close_fd_quietly(fd)
|
|
624
|
-
raise _retained_temp_error(self.display_path, temp_path.name, exc) from exc
|
|
625
|
-
published = False
|
|
626
|
-
existing_final_preserved = False
|
|
627
763
|
try:
|
|
628
764
|
with handle:
|
|
629
765
|
handle.write(data)
|
|
@@ -635,25 +771,24 @@ class _PathIdentityDestinationDirectory(DestinationDirectory):
|
|
|
635
771
|
published = True
|
|
636
772
|
else:
|
|
637
773
|
try:
|
|
638
|
-
_rename_noreplace(os.fspath(temp_path), os.fspath(final_path))
|
|
774
|
+
linked = _rename_noreplace(os.fspath(temp_path), os.fspath(final_path))
|
|
639
775
|
except FileExistsError as exc:
|
|
640
776
|
existing_final_preserved = True
|
|
641
|
-
raise DestinationDirectoryError(
|
|
642
|
-
f"Refusing to replace existing download destination {final_name!r} in {self.display_path}. "
|
|
643
|
-
"The existing final entry was preserved."
|
|
644
|
-
) from exc
|
|
777
|
+
raise DestinationDirectoryError(_existing_final_message(final_name, self.display_path)) from exc
|
|
645
778
|
published = True
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
779
|
+
assert identity is not None
|
|
780
|
+
try:
|
|
781
|
+
self._refuse_portable_alias(final_name, published_identity=identity)
|
|
782
|
+
finally:
|
|
783
|
+
if linked:
|
|
784
|
+
self._discard_linked_temp(final_name, temp_path.name, identity)
|
|
785
|
+
except BaseException as exc:
|
|
652
786
|
if published:
|
|
653
787
|
raise
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
788
|
+
replacement = self._unpublished_failure(final_name, temp_path.name, identity, exc, existing_final_preserved)
|
|
789
|
+
if replacement is not None:
|
|
790
|
+
raise replacement from exc
|
|
791
|
+
raise
|
|
657
792
|
|
|
658
793
|
def close(self) -> None:
|
|
659
794
|
self._closed = True
|