simulo 0.31.2__tar.gz → 0.33.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {simulo-0.31.2/src/simulo.egg-info → simulo-0.33.0}/PKG-INFO +6 -6
  2. {simulo-0.31.2 → simulo-0.33.0}/PYPI.md +4 -4
  3. {simulo-0.31.2 → simulo-0.33.0}/pyproject.toml +2 -2
  4. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/__init__.py +32 -2
  5. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/_runner.py +7 -0
  6. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/_secure_downloads.py +216 -81
  7. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/app.py +51 -9
  8. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/asset_api.py +1 -1
  9. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/asset_package.py +3 -3
  10. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/bundle.py +25 -18
  11. simulo-0.33.0/src/simulo/_client/cli.py +317 -0
  12. simulo-0.33.0/src/simulo/_client/cli_arguments.py +15 -0
  13. simulo-0.33.0/src/simulo/_client/cli_context.py +120 -0
  14. simulo-0.33.0/src/simulo/_client/cli_downloads.py +387 -0
  15. simulo-0.33.0/src/simulo/_client/cli_errors.py +13 -0
  16. simulo-0.33.0/src/simulo/_client/cli_job_reporting.py +733 -0
  17. simulo-0.33.0/src/simulo/_client/cli_listing.py +238 -0
  18. simulo-0.33.0/src/simulo/_client/cli_rendering.py +187 -0
  19. simulo-0.33.0/src/simulo/_client/cli_text.py +392 -0
  20. simulo-0.33.0/src/simulo/_client/cli_waiting.py +163 -0
  21. simulo-0.33.0/src/simulo/_client/commands/__init__.py +1 -0
  22. simulo-0.33.0/src/simulo/_client/commands/assets.py +2535 -0
  23. simulo-0.33.0/src/simulo/_client/commands/auth.py +252 -0
  24. simulo-0.33.0/src/simulo/_client/commands/create.py +259 -0
  25. simulo-0.33.0/src/simulo/_client/commands/jobs.py +1843 -0
  26. simulo-0.33.0/src/simulo/_client/commands/policies.py +612 -0
  27. simulo-0.33.0/src/simulo/_client/commands/recordings.py +429 -0
  28. simulo-0.33.0/src/simulo/_client/commands/run.py +1363 -0
  29. simulo-0.33.0/src/simulo/_client/commands/systems.py +132 -0
  30. simulo-0.33.0/src/simulo/_client/evaluation_report.py +618 -0
  31. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/facades.py +70 -25
  32. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/http.py +5 -2
  33. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/jobs_api.py +43 -17
  34. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/learning.py +76 -11
  35. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/packaging.py +72 -37
  36. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/policies_api.py +12 -0
  37. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/preflight_api.py +1 -4
  38. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/preflight_render.py +4 -4
  39. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/preview_report.py +64 -22
  40. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/runtime_display.py +1 -1
  41. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/submit_api.py +77 -21
  42. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/__init__.py +3 -3
  43. simulo-0.33.0/src/simulo/_client/templates/training/eval.py.tmpl +34 -0
  44. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/training/task.py.tmpl +25 -17
  45. {simulo-0.31.2 → simulo-0.33.0/src/simulo.egg-info}/PKG-INFO +6 -6
  46. {simulo-0.31.2 → simulo-0.33.0}/src/simulo.egg-info/SOURCES.txt +20 -0
  47. {simulo-0.31.2 → simulo-0.33.0}/src/simulo.egg-info/requires.txt +1 -1
  48. simulo-0.31.2/src/simulo/_client/cli.py +0 -8923
  49. {simulo-0.31.2 → simulo-0.33.0}/MANIFEST.in +0 -0
  50. {simulo-0.31.2 → simulo-0.33.0}/setup.cfg +0 -0
  51. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/__init__.py +0 -0
  52. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/_arguments.py +0 -0
  53. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/_mounts.py +0 -0
  54. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/asset.py +0 -0
  55. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/asset_pins.py +0 -0
  56. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/builtin_aliases.py +0 -0
  57. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/cancel_api.py +0 -0
  58. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/config.py +0 -0
  59. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/credentials.py +0 -0
  60. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/discovery.py +0 -0
  61. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/export_api.py +0 -0
  62. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/identity_api.py +0 -0
  63. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/install_samples.py +0 -0
  64. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/login.py +0 -0
  65. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/mode.py +0 -0
  66. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/onnx_bundle.py +0 -0
  67. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/outputs.py +0 -0
  68. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/registry.py +0 -0
  69. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/runtime.py +0 -0
  70. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/stub.py +0 -0
  71. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/training/play.py.tmpl +0 -0
  72. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/training/preview.py.tmpl +0 -0
  73. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/training/simuloignore.tmpl +0 -0
  74. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/templates/training/train.py.tmpl +0 -0
  75. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/view_fragment.py +0 -0
  76. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/view_session_api.py +0 -0
  77. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/_client/volume.py +0 -0
  78. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/callbacks.py +0 -0
  79. {simulo-0.31.2 → simulo-0.33.0}/src/simulo/py.typed +0 -0
  80. {simulo-0.31.2 → simulo-0.33.0}/src/simulo.egg-info/dependency_links.txt +0 -0
  81. {simulo-0.31.2 → simulo-0.33.0}/src/simulo.egg-info/entry_points.txt +0 -0
  82. {simulo-0.31.2 → simulo-0.33.0}/src/simulo.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: simulo
3
- Version: 0.31.2
3
+ Version: 0.33.0
4
4
  Summary: Python SDK and CLI for cloud-native robotics simulation, reinforcement learning, and robot policy training on managed GPUs.
5
5
  Author-email: Simulo Team <team@simulo.ai>
6
6
  License: BSD-3-Clause
@@ -31,7 +31,7 @@ Classifier: Topic :: System :: Distributed Computing
31
31
  Classifier: Typing :: Typed
32
32
  Requires-Python: >=3.11
33
33
  Description-Content-Type: text/markdown
34
- Requires-Dist: simulo-interfaces<0.23,>=0.22.0
34
+ Requires-Dist: simulo-interfaces<0.25,>=0.24.0
35
35
  Requires-Dist: mcap<2,>=1.3
36
36
  Requires-Dist: defusedxml>=0.7.1
37
37
  Requires-Dist: usd-core<27,>=26.5
@@ -94,7 +94,7 @@ simulo login
94
94
  cd mybot
95
95
  simulo run preview.py --frozen --strict-assets
96
96
  simulo run train.py --frozen --strict-assets
97
- simulo run play.py --policy policy_<job>:best --frozen --strict-assets
97
+ simulo run play.py --policy <policy-id>:best --frozen --strict-assets
98
98
  ```
99
99
 
100
100
  `simulo create` writes a complete starter application to your machine without
@@ -115,7 +115,7 @@ See the repository README for setup and usage.
115
115
 
116
116
  ```bash
117
117
  simulo jobs # list submitted jobs and status
118
- simulo logs --follow # stream the latest job's logs
118
+ simulo logs --follow # stream logs from your most recent job you submitted (train, play, preview or eval)
119
119
  simulo result # read the job's result
120
120
  simulo policy list # list training policies and checkpoints
121
121
  simulo policy get policy_x:best # download a verified checkpoint
@@ -126,8 +126,8 @@ simulo view # view a running --viewstream job in 3D
126
126
  simulo cancel # stop a queued or running job
127
127
  ```
128
128
 
129
- Commands that accept a job ID default to the latest job and tell you which job
130
- they selected.
129
+ Commands that accept a job ID default to the latest user-submitted job
130
+ (`train`, `play`, `preview`, or `eval`) and tell you its ID and type.
131
131
 
132
132
  ## Use versioned robot assets
133
133
 
@@ -38,7 +38,7 @@ simulo login
38
38
  cd mybot
39
39
  simulo run preview.py --frozen --strict-assets
40
40
  simulo run train.py --frozen --strict-assets
41
- simulo run play.py --policy policy_<job>:best --frozen --strict-assets
41
+ simulo run play.py --policy <policy-id>:best --frozen --strict-assets
42
42
  ```
43
43
 
44
44
  `simulo create` writes a complete starter application to your machine without
@@ -59,7 +59,7 @@ See the repository README for setup and usage.
59
59
 
60
60
  ```bash
61
61
  simulo jobs # list submitted jobs and status
62
- simulo logs --follow # stream the latest job's logs
62
+ simulo logs --follow # stream logs from your most recent job you submitted (train, play, preview or eval)
63
63
  simulo result # read the job's result
64
64
  simulo policy list # list training policies and checkpoints
65
65
  simulo policy get policy_x:best # download a verified checkpoint
@@ -70,8 +70,8 @@ simulo view # view a running --viewstream job in 3D
70
70
  simulo cancel # stop a queued or running job
71
71
  ```
72
72
 
73
- Commands that accept a job ID default to the latest job and tell you which job
74
- they selected.
73
+ Commands that accept a job ID default to the latest user-submitted job
74
+ (`train`, `play`, `preview`, or `eval`) and tell you its ID and type.
75
75
 
76
76
  ## Use versioned robot assets
77
77
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "simulo"
7
- version = "0.31.2"
7
+ version = "0.33.0"
8
8
  description = "Python SDK and CLI for cloud-native robotics simulation, reinforcement learning, and robot policy training on managed GPUs."
9
9
  readme = "PYPI.md"
10
10
  requires-python = ">=3.11"
@@ -59,7 +59,7 @@ classifiers = [
59
59
  # mcap: used by `simulo recordings` to read back a downloaded recording and
60
60
  # verify it (message count) — MIT-licensed, deps only lz4+zstandard.
61
61
  dependencies = [
62
- "simulo-interfaces>=0.22.0,<0.23",
62
+ "simulo-interfaces>=0.24.0,<0.25",
63
63
  "mcap>=1.3,<2",
64
64
  # Hardened XML parsing for URDF ingestion (USD Asset Catalogs, PR-12 fix
65
65
  # loop 2, security NIT): stdlib xml.etree.ElementTree relies on
@@ -59,7 +59,9 @@ Three kinds of name live on ``simulo``:
59
59
  prebuilt place you bring, ``World.usd(...)``.)
60
60
  """
61
61
 
62
+ import dis as _dis
62
63
  import pkgutil as _pkgutil
64
+ import sys as _sys
63
65
  from typing import TYPE_CHECKING
64
66
 
65
67
  __path__ = _pkgutil.extend_path(__path__, __name__)
@@ -143,6 +145,8 @@ from simulo.interfaces.authoring import (
143
145
  LightConfig,
144
146
  PointMarker,
145
147
  PolicyCheckpoint,
148
+ PolicyInput,
149
+ PolicyInputSchema,
146
150
  PrimitiveShape,
147
151
  RigidPhysics,
148
152
  RoughTerrain,
@@ -188,6 +192,8 @@ __all__ = [
188
192
  "JobPublicId",
189
193
  "PolicyPublicId",
190
194
  "PolicyNetworkProtocol",
195
+ "PolicyInput",
196
+ "PolicyInputSchema",
191
197
  "RecordingPublicId",
192
198
  "OutputPublicId",
193
199
  "parse_job_public_id",
@@ -234,6 +240,8 @@ __all__ = [
234
240
  "PointMarker",
235
241
  # lazy, mode-aware (the learning names + save_output)
236
242
  "ActuatorGainsConfig",
243
+ "ArticulationConfig",
244
+ "JointConfig",
237
245
  "Camera",
238
246
  "CameraSpawnConfig",
239
247
  "ContactSensor",
@@ -277,11 +285,25 @@ __all__ = [
277
285
  "Trainer",
278
286
  "Visual",
279
287
  "World",
288
+ "evaluate",
280
289
  "preview",
281
290
  "save_output",
282
291
  ]
283
292
 
284
293
 
294
+ def _is_from_import_lookup() -> bool:
295
+ """True when the running lookup is the ``IMPORT_FROM`` of ``from simulo import X``.
296
+
297
+ Attribute access and ``hasattr`` keep raising ``AttributeError``; only a ``from``
298
+ import needs the ``ImportError`` that carries the message. Any doubt answers False.
299
+ """
300
+ try:
301
+ caller = _sys._getframe(2) # skip this helper and ``__getattr__``
302
+ return _dis.opname[caller.f_code.co_code[caller.f_lasti]] == "IMPORT_FROM"
303
+ except (ValueError, IndexError):
304
+ return False
305
+
306
+
285
307
  def __getattr__(name: str) -> object:
286
308
  """PEP 562 hook: lazily resolve the mode-aware learning names.
287
309
 
@@ -298,6 +320,10 @@ def __getattr__(name: str) -> object:
298
320
  "run": "simulo.run was removed. Use `simulo run train.py` instead.",
299
321
  }
300
322
  if name in removed_hints:
323
+ if _is_from_import_lookup():
324
+ # CPython swallows an AttributeError raised during ``from simulo import X``
325
+ # and reports a bare "cannot import name", so the pointer would be lost.
326
+ raise ImportError(removed_hints[name], name=__name__)
301
327
  raise AttributeError(removed_hints[name])
302
328
  raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
303
329
 
@@ -350,11 +376,11 @@ if TYPE_CHECKING:
350
376
  # ``simulo.interfaces.authoring`` (``CameraProtocol`` / ``PropProtocol``)
351
377
  # and both are instantiated, never subclassed (``simulo.Camera(width=640,
352
378
  # ...)``, ``simulo.Prop(shape)``), so each follows the "instantiated mirror
353
- # -> facade" rule above exactly. The other 19 names
379
+ # -> facade" rule above exactly. The other names
354
380
  # have no TYPE_CHECKING alias, DELIBERATELY, per the rule's own "where none
355
381
  # exists, do not invent one" corollary:
356
382
  # - ``CameraSpawnConfig`` / ``SensorOffset`` / ``RayPattern`` /
357
- # ``ActuatorGainsConfig`` / ``GripperState`` / ``GripperCommand`` /
383
+ # ``GripperState`` / ``GripperCommand`` /
358
384
  # ``Simulation`` have no contract Protocol at all — nothing to alias
359
385
  # against.
360
386
  # - ``ContactSensor`` / ``RayCaster`` / ``IMU`` / ``ForceTorqueSensor`` /
@@ -381,6 +407,7 @@ if TYPE_CHECKING:
381
407
  from simulo._client.facades import RLPlayer as RLPlayer
382
408
  from simulo._client.facades import RLTrainer as RLTrainer
383
409
  from simulo._client.facades import Robot as Robot
410
+ from simulo._client.facades import evaluate as evaluate
384
411
  from simulo._client.facades import preview as preview
385
412
  from simulo.interfaces.authoring import SceneProtocol
386
413
  from simulo.interfaces.runtime import (
@@ -414,7 +441,10 @@ if TYPE_CHECKING:
414
441
  # ``simulo.interfaces.authoring`` class in both
415
442
  # modes, so these alias the real class — precise field/signature checking
416
443
  # (e.g. mypy rejects ``simulo.Pose(xyz="nope")``) instead of ``Any``.
444
+ from simulo.interfaces.authoring import ActuatorGainsConfig as ActuatorGainsConfig
445
+ from simulo.interfaces.authoring import ArticulationConfig as ArticulationConfig
417
446
  from simulo.interfaces.authoring import Entity as Entity
447
+ from simulo.interfaces.authoring import JointConfig as JointConfig
418
448
  from simulo.interfaces.authoring import Light as Light
419
449
  from simulo.interfaces.authoring import Material as Material
420
450
  from simulo.interfaces.authoring import Physics as Physics
@@ -92,6 +92,7 @@ class JobHandle:
92
92
  token: Optional[str] = None,
93
93
  job_type: Optional[JobType] = None,
94
94
  input_checkpoint: Optional[str] = None,
95
+ compare_checkpoint: Optional[str] = None,
95
96
  deferred_stdout: str = "",
96
97
  deferred_stderr: str = "",
97
98
  ) -> None:
@@ -104,6 +105,7 @@ class JobHandle:
104
105
  self._token = token
105
106
  self._job_type = job_type
106
107
  self._input_checkpoint = input_checkpoint
108
+ self._compare_checkpoint = compare_checkpoint
107
109
  self._deferred_stdout = deferred_stdout
108
110
  self._deferred_stderr = deferred_stderr
109
111
 
@@ -152,6 +154,11 @@ class JobHandle:
152
154
  """Policy checkpoint requested with ``simulo run --from``."""
153
155
  return self._input_checkpoint
154
156
 
157
+ @property
158
+ def compare_checkpoint(self) -> Optional[str]:
159
+ """Policy checkpoint requested with ``simulo run --compare``."""
160
+ return self._compare_checkpoint
161
+
155
162
  @property
156
163
  def deferred_stdout(self) -> str:
157
164
  """Submit-time output held until the cloud job header is printed."""
@@ -2,7 +2,7 @@
2
2
 
3
3
  THE RACE THIS CLOSES. The bulk download path validates its assembled
4
4
  filename and the selected destination directory as PATH STRINGS
5
- (``_contained_bulk_download_destination`` in ``cli.py``: resolve, then
5
+ (``_contained_bulk_download_destination`` in ``cli_downloads.py``: resolve, then
6
6
  ``relative_to``) before any bytes are fetched. The actual write — inside
7
7
  this client's checkpoint/output/recording download paths
8
8
  — re-derives the SAME destination from its path string again, after a
@@ -112,6 +112,7 @@ from __future__ import annotations
112
112
 
113
113
  import ctypes
114
114
  import errno
115
+ import logging
115
116
  import os
116
117
  import secrets
117
118
  import stat as stat_module
@@ -136,7 +137,8 @@ class DestinationDirectoryError(OSError):
136
137
 
137
138
  An ``OSError`` subclass so every existing ``except (OSError, ValueError)``
138
139
  around a download write (``_download_and_report`` and its Model/Output
139
- counterparts in ``cli.py``) already reports it as one clean CLI line;
140
+ counterparts in ``commands/jobs.py``, ``commands/policies.py``, and
141
+ ``commands/recordings.py``) already reports it as one clean CLI line;
140
142
  no caller needs a new except clause to handle this module's refusals.
141
143
  """
142
144
 
@@ -155,6 +157,38 @@ def _retained_temp_error(
155
157
  )
156
158
 
157
159
 
160
+ def _write_failure_error(
161
+ destination_display_path: Path,
162
+ final_name: str,
163
+ temp_name: str,
164
+ exc: Exception,
165
+ *,
166
+ discarded: bool,
167
+ existing_final_preserved: bool = False,
168
+ ) -> DestinationDirectoryError:
169
+ """The error for a write that failed before publication.
170
+
171
+ When the temporary entry was removed the user needs only the reason; the
172
+ retained-entry wording is for the case where it could not be removed.
173
+ """
174
+ if not discarded:
175
+ return _retained_temp_error(
176
+ destination_display_path, temp_name, exc, existing_final_preserved=existing_final_preserved
177
+ )
178
+ if isinstance(exc, DestinationDirectoryError):
179
+ return exc
180
+ return DestinationDirectoryError(
181
+ f"Could not write {final_name!r} into {destination_display_path}: {exc}. " "The partial download was removed."
182
+ )
183
+
184
+
185
+ def _existing_final_message(final_name: str, destination_display_path: Path) -> str:
186
+ return (
187
+ f"{final_name!r} already exists in {destination_display_path}; "
188
+ "remove it, or download it to another folder with -o DIR."
189
+ )
190
+
191
+
158
192
  def _close_fd_quietly(fd: int) -> None:
159
193
  try:
160
194
  os.close(fd)
@@ -194,8 +228,8 @@ _TEMP_NAME_ATTEMPTS = 64
194
228
  # or an existing final entry is left untouched. ``link`` followed by
195
229
  # ``unlink`` cannot offer that postcondition because a process death between
196
230
  # those calls leaves both names behind. Keep the optional libc binding at
197
- # module scope so unsupported libc/platform combinations are explicitly
198
- # refused rather than silently falling back to replacement.
231
+ # module scope; a libc, platform or filesystem without the primitive falls back
232
+ # to :func:`_publish_by_link`, which still never replaces an existing file.
199
233
  _AT_FDCWD = -100
200
234
  _RENAME_NOREPLACE = 1
201
235
  _RENAME_EXCL = 0x00000004
@@ -221,14 +255,16 @@ def _rename_noreplace(
221
255
  *,
222
256
  src_dir_fd: Optional[int] = None,
223
257
  dst_dir_fd: Optional[int] = None,
224
- ) -> None:
258
+ ) -> bool:
225
259
  """Atomically publish *source* at absent *destination*, or fail closed.
226
260
 
227
261
  Linux uses ``renameat2(RENAME_NOREPLACE)`` and macOS uses
228
262
  ``renameatx_np(RENAME_EXCL)``. Windows' native ``os.rename`` refuses an
229
- existing destination. Other platforms deliberately refuse the
230
- server-derived no-clobber path instead of using a two-name link/unlink
231
- sequence or replacing an existing file. Explicit user file targets keep
263
+ existing destination. Where the platform, or the filesystem under the
264
+ destination (an NTFS or FAT mount), has no such primitive,
265
+ :func:`_publish_by_link` publishes without ever replacing an existing
266
+ file. Return ``True`` when linking leaves the temporary name for the caller
267
+ to discard after recording publication. Explicit user file targets keep
232
268
  using ``os.replace`` through ``replace_existing=True``.
233
269
  """
234
270
  if os.name == "nt":
@@ -237,7 +273,7 @@ def _rename_noreplace(
237
273
  "This platform cannot atomically publish a server-named download through directory descriptors."
238
274
  )
239
275
  os.rename(source, destination)
240
- return
276
+ return False
241
277
  if sys.platform.startswith("linux"):
242
278
  rename_function = _renameat2
243
279
  flag = _RENAME_NOREPLACE
@@ -248,9 +284,8 @@ def _rename_noreplace(
248
284
  rename_function = None
249
285
  flag = 0
250
286
  if rename_function is None:
251
- raise DestinationDirectoryError(
252
- "This platform cannot atomically publish a server-named download without replacing an existing file."
253
- )
287
+ _publish_by_link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd)
288
+ return True
254
289
  old_dir_fd = _AT_FDCWD if src_dir_fd is None else src_dir_fd
255
290
  new_dir_fd = _AT_FDCWD if dst_dir_fd is None else dst_dir_fd
256
291
  ctypes.set_errno(0)
@@ -262,17 +297,43 @@ def _rename_noreplace(
262
297
  flag,
263
298
  )
264
299
  if result == 0:
265
- return
300
+ return False
266
301
  error_number = ctypes.get_errno()
267
302
  if error_number == errno.EEXIST:
268
303
  raise FileExistsError(error_number, os.strerror(error_number), destination)
269
- if error_number in (errno.EINVAL, errno.ENOSYS, errno.EOPNOTSUPP):
270
- raise DestinationDirectoryError(
271
- "This filesystem cannot atomically publish a server-named download without replacing an existing file."
272
- )
304
+ if error_number in (errno.EINVAL, errno.ENOSYS, errno.EOPNOTSUPP, errno.ENOTSUP):
305
+ # The filesystem (an NTFS or FAT mount, a network share) does not
306
+ # implement the no-replace flag.
307
+ _publish_by_link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd)
308
+ return True
273
309
  raise OSError(error_number, os.strerror(error_number), destination)
274
310
 
275
311
 
312
+ def _publish_by_link(
313
+ source: str,
314
+ destination: str,
315
+ *,
316
+ src_dir_fd: Optional[int] = None,
317
+ dst_dir_fd: Optional[int] = None,
318
+ ) -> None:
319
+ """Publish with an atomic no-clobber hard link.
320
+
321
+ Unlike an exclusive-create claim followed by replacement, ``link`` refuses
322
+ any occupied destination at publication time. The caller discards the
323
+ temporary name after recording publication, checking its identity first.
324
+ Filesystems without hard links are refused rather than risking replacement.
325
+ """
326
+ try:
327
+ os.link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd, follow_symlinks=False)
328
+ except FileExistsError:
329
+ raise
330
+ except (OSError, NotImplementedError) as exc:
331
+ raise DestinationDirectoryError(
332
+ "This filesystem cannot safely publish a download without replacing an existing file: "
333
+ f"the atomic hard link failed ({exc}). Choose another download folder with -o DIR."
334
+ ) from exc
335
+
336
+
276
337
  def _random_temp_name(final_name: str) -> str:
277
338
  """One candidate temp-file name for ``final_name``, unpredictable
278
339
  enough that an attacker cannot pre-plant a symlink at it — the
@@ -362,19 +423,62 @@ class DestinationDirectory:
362
423
  A same-directory temporary file is written, flushed, and ``fsync``'d,
363
424
  then published atomically. With ``replace_existing=True`` it replaces
364
425
  a prior complete entry; otherwise publication fails if *any* final
365
- entry already exists. Any failure after temporary-file creation may
366
- leave that mode-0600 ``.part`` entry behind. No backend deletes it
367
- automatically because a pathname can be replaced before cleanup.
368
- Ordinary write and publication errors name the temporary entry and ask
369
- the operator to locate and verify it before manual cleanup.
370
- ``KeyboardInterrupt`` and
371
- ``SystemExit`` are re-raised unchanged; their possible residue is
372
- documented here rather than reported while interrupting the caller.
426
+ entry already exists. After any failure before publication the
427
+ temporary entry is removed, but only when the name still refers to the
428
+ very file this call created (same device and inode, and for the
429
+ path-based writer the same directory): a pathname can be replaced
430
+ before cleanup, and an entry that is no longer provably ours is left
431
+ alone. Then the error says plainly why the write failed. Only when
432
+ that entry cannot be removed does the error name the retained
433
+ mode-0600 ``.part`` entry and ask the operator to locate and verify it
434
+ before manual cleanup. ``KeyboardInterrupt`` and ``SystemExit``
435
+ remove the entry the same way and are re-raised unchanged.
436
+ When publication uses a hard link, the completed temporary name is
437
+ removed using the same identity check; failed cleanup is reported.
373
438
  ``final_name`` must be a bare basename (no path separators); see
374
439
  :func:`_validate_leaf_name`.
375
440
  """
376
441
  raise NotImplementedError
377
442
 
443
+ def _discard_temp(self, temp_name: str, identity: Optional[tuple[int, int]]) -> bool:
444
+ raise NotImplementedError
445
+
446
+ def _discard_linked_temp(self, final_name: str, temp_name: str, identity: tuple[int, int]) -> None:
447
+ if not self._discard_temp(temp_name, identity):
448
+ logging.getLogger(__name__).warning(
449
+ "Download %r was published in %s. Temporary entry %r could not be safely removed. "
450
+ "Verify its identity before manual cleanup.",
451
+ final_name,
452
+ self.display_path,
453
+ temp_name,
454
+ )
455
+
456
+ def _unpublished_failure(
457
+ self,
458
+ final_name: str,
459
+ temp_name: str,
460
+ identity: Optional[tuple[int, int]],
461
+ exc: BaseException,
462
+ existing_final_preserved: bool,
463
+ ) -> Optional[DestinationDirectoryError]:
464
+ """Remove the unpublished temp entry; return the error to raise instead of *exc*.
465
+
466
+ ``None`` means re-raise *exc* unchanged: an interrupt or exit, or an
467
+ error that already says what it needs to.
468
+ """
469
+ discarded = self._discard_temp(temp_name, identity)
470
+ if not isinstance(exc, Exception):
471
+ return None
472
+ error = _write_failure_error(
473
+ self.display_path,
474
+ final_name,
475
+ temp_name,
476
+ exc,
477
+ discarded=discarded,
478
+ existing_final_preserved=existing_final_preserved,
479
+ )
480
+ return None if error is exc else error
481
+
378
482
  def close(self) -> None:
379
483
  raise NotImplementedError
380
484
 
@@ -455,29 +559,45 @@ class _DirFdDestinationDirectory(DestinationDirectory):
455
559
  f"Refusing to publish {final_name!r}: portable-equivalent alias {aliases[0]!r} exists."
456
560
  )
457
561
 
562
+ def _discard_temp(self, temp_name: str, identity: Optional[tuple[int, int]]) -> bool:
563
+ """Remove the temporary entry this call created; ``True`` when it is gone.
564
+
565
+ The held directory descriptor pins the directory, so only the entry's
566
+ name can have changed. It is removed only while it still has the
567
+ identity of the file this call created. A name that now refers to
568
+ anything else is left alone.
569
+ """
570
+ try:
571
+ if identity is None:
572
+ return False
573
+ current = os.stat(temp_name, dir_fd=self._fd, follow_symlinks=False)
574
+ if (current.st_dev, current.st_ino) != identity:
575
+ return False
576
+ os.unlink(temp_name, dir_fd=self._fd)
577
+ except FileNotFoundError:
578
+ return True
579
+ except OSError:
580
+ return False
581
+ return True
582
+
458
583
  def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
459
584
  _validate_leaf_name(final_name)
460
585
  if not replace_existing:
461
586
  self._refuse_portable_alias(final_name)
462
587
  fd, temp_name = self._mkstemp(final_name)
588
+ identity: Optional[tuple[int, int]] = None
589
+ published = False
590
+ existing_final_preserved = False
463
591
  try:
464
- published_identity = os.fstat(fd)
465
- except (KeyboardInterrupt, SystemExit):
466
- _close_fd_quietly(fd)
467
- raise
468
- except Exception as exc:
469
- _close_fd_quietly(fd)
470
- raise _retained_temp_error(self.display_path, temp_name, exc) from exc
471
- try:
592
+ created = os.fstat(fd)
593
+ identity = (created.st_dev, created.st_ino)
472
594
  handle = os.fdopen(fd, "wb")
473
- except (KeyboardInterrupt, SystemExit):
595
+ except BaseException as exc:
474
596
  _close_fd_quietly(fd)
597
+ replacement = self._unpublished_failure(final_name, temp_name, identity, exc, existing_final_preserved)
598
+ if replacement is not None:
599
+ raise replacement from exc
475
600
  raise
476
- except Exception as exc:
477
- _close_fd_quietly(fd)
478
- raise _retained_temp_error(self.display_path, temp_name, exc) from exc
479
- published = False
480
- existing_final_preserved = False
481
601
  try:
482
602
  with handle:
483
603
  handle.write(data)
@@ -488,25 +608,24 @@ class _DirFdDestinationDirectory(DestinationDirectory):
488
608
  published = True
489
609
  else:
490
610
  try:
491
- _rename_noreplace(temp_name, final_name, src_dir_fd=self._fd, dst_dir_fd=self._fd)
611
+ linked = _rename_noreplace(temp_name, final_name, src_dir_fd=self._fd, dst_dir_fd=self._fd)
492
612
  except FileExistsError as exc:
493
613
  existing_final_preserved = True
494
- raise DestinationDirectoryError(
495
- f"Refusing to replace existing download destination {final_name!r} in {self.display_path}. "
496
- "The existing final entry was preserved."
497
- ) from exc
614
+ raise DestinationDirectoryError(_existing_final_message(final_name, self.display_path)) from exc
498
615
  published = True
499
- self._refuse_portable_alias(
500
- final_name, published_identity=(published_identity.st_dev, published_identity.st_ino)
501
- )
502
- except (KeyboardInterrupt, SystemExit):
503
- raise
504
- except Exception as exc:
616
+ assert identity is not None
617
+ try:
618
+ self._refuse_portable_alias(final_name, published_identity=identity)
619
+ finally:
620
+ if linked:
621
+ self._discard_linked_temp(final_name, temp_name, identity)
622
+ except BaseException as exc:
505
623
  if published:
506
624
  raise
507
- raise _retained_temp_error(
508
- self.display_path, temp_name, exc, existing_final_preserved=existing_final_preserved
509
- ) from exc
625
+ replacement = self._unpublished_failure(final_name, temp_name, identity, exc, existing_final_preserved)
626
+ if replacement is not None:
627
+ raise replacement from exc
628
+ raise
510
629
 
511
630
  def close(self) -> None:
512
631
  if not self._closed:
@@ -599,31 +718,48 @@ class _PathIdentityDestinationDirectory(DestinationDirectory):
599
718
  )
600
719
  raise DestinationDirectoryError(f"Refusing to publish {final_name!r}: portable-equivalent alias exists.")
601
720
 
721
+ def _discard_temp(self, temp_name: str, identity: Optional[tuple[int, int]]) -> bool:
722
+ """Remove the temporary entry this call created; ``True`` when it is gone.
723
+
724
+ Nothing holds this directory, so the directory is re-verified first and
725
+ the entry is removed only while it still has the identity of the file
726
+ this call created. Anything else is left alone.
727
+ """
728
+ try:
729
+ if identity is None:
730
+ return False
731
+ self._verify_unchanged()
732
+ temp_path = os.fspath(self.display_path / temp_name)
733
+ current = os.stat(temp_path, follow_symlinks=False)
734
+ if (current.st_dev, current.st_ino) != identity:
735
+ return False
736
+ os.unlink(temp_path)
737
+ except FileNotFoundError:
738
+ return True
739
+ except OSError:
740
+ return False
741
+ return True
742
+
602
743
  def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
603
744
  _validate_leaf_name(final_name)
604
745
  self._verify_unchanged()
605
746
  if not replace_existing:
606
747
  self._refuse_portable_alias(final_name)
607
748
  fd, temp_path = self._mkstemp(final_name)
608
- try:
609
- published_identity = os.fstat(fd)
610
- except (KeyboardInterrupt, SystemExit):
611
- _close_fd_quietly(fd)
612
- raise
613
- except Exception as exc:
614
- _close_fd_quietly(fd)
615
- raise _retained_temp_error(self.display_path, temp_path.name, exc) from exc
749
+ identity: Optional[tuple[int, int]] = None
616
750
  final_path = self.display_path / final_name
751
+ published = False
752
+ existing_final_preserved = False
617
753
  try:
754
+ created = os.fstat(fd)
755
+ identity = (created.st_dev, created.st_ino)
618
756
  handle = os.fdopen(fd, "wb")
619
- except (KeyboardInterrupt, SystemExit):
757
+ except BaseException as exc:
620
758
  _close_fd_quietly(fd)
759
+ replacement = self._unpublished_failure(final_name, temp_path.name, identity, exc, existing_final_preserved)
760
+ if replacement is not None:
761
+ raise replacement from exc
621
762
  raise
622
- except Exception as exc:
623
- _close_fd_quietly(fd)
624
- raise _retained_temp_error(self.display_path, temp_path.name, exc) from exc
625
- published = False
626
- existing_final_preserved = False
627
763
  try:
628
764
  with handle:
629
765
  handle.write(data)
@@ -635,25 +771,24 @@ class _PathIdentityDestinationDirectory(DestinationDirectory):
635
771
  published = True
636
772
  else:
637
773
  try:
638
- _rename_noreplace(os.fspath(temp_path), os.fspath(final_path))
774
+ linked = _rename_noreplace(os.fspath(temp_path), os.fspath(final_path))
639
775
  except FileExistsError as exc:
640
776
  existing_final_preserved = True
641
- raise DestinationDirectoryError(
642
- f"Refusing to replace existing download destination {final_name!r} in {self.display_path}. "
643
- "The existing final entry was preserved."
644
- ) from exc
777
+ raise DestinationDirectoryError(_existing_final_message(final_name, self.display_path)) from exc
645
778
  published = True
646
- self._refuse_portable_alias(
647
- final_name, published_identity=(published_identity.st_dev, published_identity.st_ino)
648
- )
649
- except (KeyboardInterrupt, SystemExit):
650
- raise
651
- except Exception as exc:
779
+ assert identity is not None
780
+ try:
781
+ self._refuse_portable_alias(final_name, published_identity=identity)
782
+ finally:
783
+ if linked:
784
+ self._discard_linked_temp(final_name, temp_path.name, identity)
785
+ except BaseException as exc:
652
786
  if published:
653
787
  raise
654
- raise _retained_temp_error(
655
- self.display_path, temp_path.name, exc, existing_final_preserved=existing_final_preserved
656
- ) from exc
788
+ replacement = self._unpublished_failure(final_name, temp_path.name, identity, exc, existing_final_preserved)
789
+ if replacement is not None:
790
+ raise replacement from exc
791
+ raise
657
792
 
658
793
  def close(self) -> None:
659
794
  self._closed = True