simulo 0.26.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. simulo/__init__.py +433 -0
  2. simulo/_client/__init__.py +6 -0
  3. simulo/_client/_entrypoint.py +313 -0
  4. simulo/_client/_mounts.py +25 -0
  5. simulo/_client/_runner.py +186 -0
  6. simulo/_client/_secure_downloads.py +1181 -0
  7. simulo/_client/app.py +1308 -0
  8. simulo/_client/asset.py +331 -0
  9. simulo/_client/asset_api.py +517 -0
  10. simulo/_client/asset_package.py +1103 -0
  11. simulo/_client/asset_pins.py +187 -0
  12. simulo/_client/builtin_aliases.py +107 -0
  13. simulo/_client/bundle.py +254 -0
  14. simulo/_client/cancel_api.py +104 -0
  15. simulo/_client/cli.py +9063 -0
  16. simulo/_client/config.py +186 -0
  17. simulo/_client/credentials.py +210 -0
  18. simulo/_client/discovery.py +214 -0
  19. simulo/_client/export_api.py +212 -0
  20. simulo/_client/export_bundle.py +296 -0
  21. simulo/_client/facades.py +581 -0
  22. simulo/_client/http.py +414 -0
  23. simulo/_client/identity_api.py +117 -0
  24. simulo/_client/install_samples.py +267 -0
  25. simulo/_client/jobs_api.py +224 -0
  26. simulo/_client/learning.py +393 -0
  27. simulo/_client/login.py +319 -0
  28. simulo/_client/mode.py +29 -0
  29. simulo/_client/outputs.py +116 -0
  30. simulo/_client/packaging.py +445 -0
  31. simulo/_client/preflight_api.py +186 -0
  32. simulo/_client/preflight_render.py +200 -0
  33. simulo/_client/registry.py +98 -0
  34. simulo/_client/runtime.py +185 -0
  35. simulo/_client/runtime_display.py +90 -0
  36. simulo/_client/seed_ref.py +76 -0
  37. simulo/_client/stub.py +41 -0
  38. simulo/_client/submit_api.py +1057 -0
  39. simulo/_client/templates/__init__.py +21 -0
  40. simulo/_client/templates/inference/app.py.tmpl +316 -0
  41. simulo/_client/templates/inference/simuloignore.tmpl +30 -0
  42. simulo/_client/templates/scenario/app.py.tmpl +93 -0
  43. simulo/_client/templates/scenario/simuloignore.tmpl +27 -0
  44. simulo/_client/templates/training/app.py.tmpl +235 -0
  45. simulo/_client/templates/training/simuloignore.tmpl +29 -0
  46. simulo/_client/view_fragment.py +21 -0
  47. simulo/_client/view_session_api.py +122 -0
  48. simulo/_client/volume.py +71 -0
  49. simulo/callbacks.py +274 -0
  50. simulo/py.typed +0 -0
  51. simulo-0.26.0.dist-info/METADATA +130 -0
  52. simulo-0.26.0.dist-info/RECORD +55 -0
  53. simulo-0.26.0.dist-info/WHEEL +5 -0
  54. simulo-0.26.0.dist-info/entry_points.txt +2 -0
  55. simulo-0.26.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1181 @@
1
+ """Race-resistant destination-directory handling for CLI downloads.
2
+
3
+ THE RACE THIS CLOSES. The bulk download path validates its assembled
4
+ filename and the selected destination directory as PATH STRINGS
5
+ (``_contained_bulk_download_destination`` in ``cli.py``: resolve, then
6
+ ``relative_to``) before any bytes are fetched. The actual write — inside
7
+ this client's ``download_model``/``download_output``/``download_recording``
8
+ — re-derives the SAME destination from its path string again, after a
9
+ network fetch: an ``exist_ok=True`` ``mkdir`` (which silently ACCEPTS a
10
+ symlink planted at a previously-missing path, so long as it resolves to a
11
+ directory), a SEPARATE ``tempfile.mkstemp(dir=...)`` call, then a SEPARATE
12
+ ``os.replace`` call. A local attacker who can create files in whatever
13
+ directory CONTAINS the selected destination — or in any ANCESTOR of it a
14
+ nested ``-o a/b/c`` would create — can replace a previously-missing
15
+ directory with a symlink at any point before the eventual write and
16
+ redirect the downloaded bytes outside the approved destination. This was
17
+ named, and explicitly deferred, by the security review of the change that
18
+ landed the response-ID, filename, normalization, and existing-symlink-escape
19
+ containment this module extends.
20
+
21
+ THE FIX is two-phase, and the split matters: **observe, then open.**
22
+
23
+ :func:`observe_destination` runs FIRST — before any network I/O for the
24
+ download begins — and walks the ENTIRE destination path, component by
25
+ component, from a trusted anchor (filesystem root for an absolute path, the
26
+ process's current working directory for a relative one) down to the leaf.
27
+ For the PREFIX of components that already exist, it does not merely record
28
+ their (device, inode) VALUES — an inode number is a value a filesystem
29
+ allocator can and does reuse (measured on ext4: a directory removed with
30
+ ``rmdir`` frees its inode for the very next ``mkdir`` in the same parent, on
31
+ the first attempt), so a value comparison taken later can be fooled by an
32
+ attacker who deletes the observed directory and lets a NEW directory they
33
+ control claim the same number before the eventual check runs. Instead, the
34
+ DEEPEST existing component's file descriptor is kept OPEN — held inside the
35
+ returned :class:`DestinationObservation` — which pins its inode: the kernel
36
+ will not hand that inode number to a new allocation while any process holds
37
+ it open, so the value-reuse attack has no purchase on it. Every component
38
+ AFTER that point does not exist yet; only its NAME is recorded, nothing
39
+ about its contents is trusted. This call is read-only for anything it does
40
+ not already hold open, so a subsequent digest mismatch or failed fetch
41
+ still leaves zero NEW filesystem footprint, exactly as before.
42
+
43
+ :func:`open_destination_directory` runs SECOND — immediately before the
44
+ atomic write, once bytes are ready to land — and takes that observation as
45
+ a REQUIRED argument. On POSIX it does NOT re-walk the existing prefix at
46
+ all: it takes OVER the held file descriptor directly (ownership transfers
47
+ to the returned handle) and continues the walk, ``dir_fd``-chained, only
48
+ for the components the observation recorded as absent — creating each with
49
+ a bare ``os.mkdir`` (which fails outright, symlink or not, if anything
50
+ already occupies the name) and then opening it ``O_NOFOLLOW``, so anything
51
+ other than the plain directory just created is provably a race (nothing was
52
+ there when the operation began) and is refused rather than followed.
53
+
54
+ Because the existing prefix is never re-derived from a path string —
55
+ ``open_destination_directory`` continues from the SAME open descriptor
56
+ ``observe_destination`` acquired, never a fresh lookup of the same name —
57
+ and every absent component is only ever created under ``O_NOFOLLOW``,
58
+ nothing that happens during the network fetch can redirect the eventual
59
+ write, for EVERY component of the destination this module creates or
60
+ holds, not only the final one.
61
+
62
+ MEASURED RESIDUAL. The gap between ``observe_destination`` and
63
+ ``open_destination_directory`` is NOT small: callers call the first before
64
+ even resolving a presigned URL and the second only after the network fetch
65
+ (and any digest verification) completes — the gap IS the fetch's entire,
66
+ possibly long, duration, deliberately, because a digest mismatch or a
67
+ failed fetch must still leave zero new filesystem footprint. The invariant
68
+ this module provides does not depend on that gap's width. On POSIX, the
69
+ existing prefix is bound to a held file descriptor from the moment it is
70
+ observed, so nothing that happens to its NAME during the gap — deletion,
71
+ inode reuse at the freed number, a symlink planted at the freed name — can
72
+ change what the returned handle operates on, because the handle never
73
+ looks that name up again; and every component recorded absent is only ever
74
+ created under ``O_NOFOLLOW`` at open time, so a plant during the gap is
75
+ refused there instead. What is genuinely out of scope: the trusted anchor
76
+ itself (filesystem root, or the process's own current working directory),
77
+ already outside this module's and the CLI's threat model, and the CALLER's
78
+ own responsibility to pass ``observe_destination`` and
79
+ ``open_destination_directory`` the intended destination in the first place
80
+ (a caller bug there is not an attacker capability this module can guard).
81
+
82
+ WINDOWS. CPython's ``os`` module does not expose ``dir_fd``-relative
83
+ operations on Windows for any primitive this module needs — ``os.open``,
84
+ ``os.mkdir``, and ``os.replace`` are all documented "Availability: Unix" for
85
+ their ``dir_fd``/``*_dir_fd`` parameters, and ``os.supports_dir_fd`` is
86
+ empty there — so neither the held-descriptor technique nor the
87
+ ``dir_fd``-chained walk is available.
88
+ ``_PathIdentityDestinationDirectory`` still applies a per-component
89
+ create-or-verify treatment to every component, ancestors included, but each
90
+ check is its own separate ``os.stat``/``os.mkdir`` call against an
91
+ ACCUMULATED path string, never a held handle — so a swap of any EARLIER
92
+ component, at any point after that component's own check already ran,
93
+ redirects every SUBSEQUENT component's ``mkdir``/``stat`` for the
94
+ REMAINDER of that walk, not merely "the next syscall on that same
95
+ component". This is a real, disclosed, non-zero-width gap the POSIX path
96
+ does not have. Windows also has a primitive Linux/macOS do not: DIRECTORY
97
+ JUNCTIONS, created via ``mklink /J`` with NO elevated privilege (unlike a
98
+ true Windows symlink, which needs ``SeCreateSymbolicLinkPrivilege``) and
99
+ NOT detected by ``stat.S_ISLNK``. This module checks ``st_reparse_tag``
100
+ explicitly wherever it checks for a symlink on the fallback path, so the
101
+ unprivileged primitive an attacker can actually use is the one being
102
+ guarded, not only the privileged one they usually cannot. One user-visible
103
+ consequence: a PRE-EXISTING junction as the selected destination is now
104
+ refused outright, the same as a freshly-planted one, rather than silently
105
+ followed — junctions are the ordinary UNPRIVILEGED way Windows users link
106
+ directories day to day, so a ``-o`` target that is a junction, which
107
+ worked before this module existed, now gets a hard error where it did not
108
+ before.
109
+ """
110
+
111
+ from __future__ import annotations
112
+
113
+ import ctypes
114
+ import errno
115
+ import os
116
+ import secrets
117
+ import stat as stat_module
118
+ import sys
119
+ import unicodedata
120
+ from pathlib import Path
121
+ from typing import Optional
122
+
123
+ __all__ = [
124
+ "DestinationDirectory",
125
+ "DestinationDirectoryError",
126
+ "DestinationObservation",
127
+ "SUPPORTS_DIR_FD",
128
+ "observe_destination",
129
+ "open_destination_directory",
130
+ ]
131
+
132
+
133
+ class DestinationDirectoryError(OSError):
134
+ """A download destination could not be safely opened, created, or
135
+ written to — including a fail-closed refusal of a symlink/junction race.
136
+
137
+ An ``OSError`` subclass so every existing ``except (OSError, ValueError)``
138
+ around a download write (``_download_and_report`` and its Model/Output
139
+ counterparts in ``cli.py``) already reports it as one clean CLI line;
140
+ no caller needs a new except clause to handle this module's refusals.
141
+ """
142
+
143
+
144
+ def _retained_temp_error(
145
+ destination_display_path: Path, temp_name: str, exc: Exception, *, existing_final_preserved: bool = False
146
+ ) -> DestinationDirectoryError:
147
+ """Describe a possible temporary residue without trusting its current path."""
148
+ preservation = "Existing final entry was preserved. " if existing_final_preserved else ""
149
+ return DestinationDirectoryError(
150
+ f"{preservation}Download created private temporary entry {temp_name!r} in the originally selected "
151
+ f"destination directory {destination_display_path} before the failure. That directory and name may now "
152
+ "resolve elsewhere; locate and verify the original entry before cleanup. Do not delete a same-named "
153
+ "current path unless its identity is independently confirmed. "
154
+ f"Could not complete the download write: {exc}."
155
+ )
156
+
157
+
158
+ def _close_fd_quietly(fd: int) -> None:
159
+ try:
160
+ os.close(fd)
161
+ except OSError:
162
+ pass
163
+
164
+
165
+ #: ``True`` when this interpreter can perform openat-style,
166
+ #: directory-file-descriptor-relative operations for every primitive this
167
+ #: module needs. A platform property, checked once at import time — used by
168
+ #: :func:`open_destination_directory` to pick
169
+ #: :class:`_DirFdDestinationDirectory` (POSIX, race-free) or
170
+ #: :class:`_PathIdentityDestinationDirectory` (the documented Windows
171
+ #: fallback, race-narrowing) — see the module docstring's WINDOWS section.
172
+ SUPPORTS_DIR_FD = (
173
+ hasattr(os, "O_DIRECTORY")
174
+ and hasattr(os, "O_NOFOLLOW")
175
+ and os.open in os.supports_dir_fd
176
+ and os.mkdir in os.supports_dir_fd
177
+ # `os.replace` itself is never listed in `os.supports_dir_fd` — CPython
178
+ # only advertises its sibling `os.rename` there, even though (measured:
179
+ # both share the same underlying `posixmodule.c` rename implementation,
180
+ # differing only in overwrite behavior) `os.replace(src, dst,
181
+ # src_dir_fd=..., dst_dir_fd=...)` works everywhere `os.rename` with
182
+ # those same arguments does. Check the one CPython actually documents.
183
+ and os.rename in os.supports_dir_fd
184
+ and os.unlink in os.supports_dir_fd
185
+ )
186
+
187
+ #: Bounded retries for picking an unused random temp-file name directly
188
+ #: inside the destination directory — mirrors ``tempfile``'s own retry
189
+ #: count for the same collision-on-``O_EXCL`` situation.
190
+ _TEMP_NAME_ATTEMPTS = 64
191
+
192
+ # Linux ``renameat2(..., RENAME_NOREPLACE)`` is one syscall: either the
193
+ # temporary entry becomes the final entry and the temporary name disappears,
194
+ # or an existing final entry is left untouched. ``link`` followed by
195
+ # ``unlink`` cannot offer that postcondition because a process death between
196
+ # those calls leaves both names behind. Keep the optional libc binding at
197
+ # module scope so unsupported libc/platform combinations are explicitly
198
+ # refused rather than silently falling back to replacement.
199
+ _AT_FDCWD = -100
200
+ _RENAME_NOREPLACE = 1
201
+ _RENAME_EXCL = 0x00000004
202
+ _renameat2 = None
203
+ _renameatx_np = None
204
+ if sys.platform.startswith("linux"):
205
+ _libc = ctypes.CDLL(None, use_errno=True)
206
+ _renameat2 = getattr(_libc, "renameat2", None)
207
+ if _renameat2 is not None:
208
+ _renameat2.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
209
+ _renameat2.restype = ctypes.c_int
210
+ elif sys.platform == "darwin":
211
+ _libc = ctypes.CDLL(None, use_errno=True)
212
+ _renameatx_np = getattr(_libc, "renameatx_np", None)
213
+ if _renameatx_np is not None:
214
+ _renameatx_np.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
215
+ _renameatx_np.restype = ctypes.c_int
216
+
217
+
218
+ def _rename_noreplace(
219
+ source: str,
220
+ destination: str,
221
+ *,
222
+ src_dir_fd: Optional[int] = None,
223
+ dst_dir_fd: Optional[int] = None,
224
+ ) -> None:
225
+ """Atomically publish *source* at absent *destination*, or fail closed.
226
+
227
+ Linux uses ``renameat2(RENAME_NOREPLACE)`` and macOS uses
228
+ ``renameatx_np(RENAME_EXCL)``. Windows' native ``os.rename`` refuses an
229
+ existing destination. Other platforms deliberately refuse the
230
+ server-derived no-clobber path instead of using a two-name link/unlink
231
+ sequence or replacing an existing file. Explicit user file targets keep
232
+ using ``os.replace`` through ``replace_existing=True``.
233
+ """
234
+ if os.name == "nt":
235
+ if src_dir_fd is not None or dst_dir_fd is not None:
236
+ raise DestinationDirectoryError(
237
+ "This platform cannot atomically publish a server-named download through directory descriptors."
238
+ )
239
+ os.rename(source, destination)
240
+ return
241
+ if sys.platform.startswith("linux"):
242
+ rename_function = _renameat2
243
+ flag = _RENAME_NOREPLACE
244
+ elif sys.platform == "darwin":
245
+ rename_function = _renameatx_np
246
+ flag = _RENAME_EXCL
247
+ else:
248
+ rename_function = None
249
+ flag = 0
250
+ if rename_function is None:
251
+ raise DestinationDirectoryError(
252
+ "This platform cannot atomically publish a server-named download without replacing an existing file."
253
+ )
254
+ old_dir_fd = _AT_FDCWD if src_dir_fd is None else src_dir_fd
255
+ new_dir_fd = _AT_FDCWD if dst_dir_fd is None else dst_dir_fd
256
+ ctypes.set_errno(0)
257
+ result = rename_function(
258
+ old_dir_fd,
259
+ os.fsencode(source),
260
+ new_dir_fd,
261
+ os.fsencode(destination),
262
+ flag,
263
+ )
264
+ if result == 0:
265
+ return
266
+ error_number = ctypes.get_errno()
267
+ if error_number == errno.EEXIST:
268
+ raise FileExistsError(error_number, os.strerror(error_number), destination)
269
+ if error_number in (errno.EINVAL, errno.ENOSYS, errno.EOPNOTSUPP):
270
+ raise DestinationDirectoryError(
271
+ "This filesystem cannot atomically publish a server-named download without replacing an existing file."
272
+ )
273
+ raise OSError(error_number, os.strerror(error_number), destination)
274
+
275
+
276
+ def _random_temp_name(final_name: str) -> str:
277
+ """One candidate temp-file name for ``final_name``, unpredictable
278
+ enough that an attacker cannot pre-plant a symlink at it — the
279
+ surrounding directory's OWN identity is what this module protects;
280
+ the temp name's job is only to avoid colliding with anything already
281
+ there, exactly like ``tempfile.mkstemp``."""
282
+ return f".{final_name}.{secrets.token_hex(8)}.part"
283
+
284
+
285
+ def _portable_name_key(name: str) -> str:
286
+ return unicodedata.normalize("NFKC", name).casefold()
287
+
288
+
289
+ def _validate_leaf_name(final_name: str) -> None:
290
+ """Refuse anything that is not a bare basename.
291
+
292
+ Every documented caller already gets a bare basename from
293
+ ``_safe_download_filename`` before it ever reaches here, but the
294
+ containment this module sells ("cannot land outside the directory the
295
+ handle identifies, by construction") is only true if this precondition
296
+ actually holds — checked explicitly rather than left as an unenforced
297
+ assumption a future caller could violate silently.
298
+ """
299
+ if not final_name or final_name in (".", "..") or "/" in final_name:
300
+ raise DestinationDirectoryError(f"Refusing to write {final_name!r}: not a safe bare filename.")
301
+ if os.sep != "/" and os.sep in final_name:
302
+ raise DestinationDirectoryError(f"Refusing to write {final_name!r}: not a safe bare filename.")
303
+ if os.altsep and os.altsep in final_name:
304
+ raise DestinationDirectoryError(f"Refusing to write {final_name!r}: not a safe bare filename.")
305
+
306
+
307
+ def _is_reparse_point(info: os.stat_result) -> bool:
308
+ """``True`` for a Windows reparse point — a directory JUNCTION
309
+ (``mklink /J``, no elevated privilege required) among them.
310
+
311
+ ``stat.S_ISLNK`` does not cover a junction (it is not the "symlink"
312
+ reparse type NTFS uses), so a check that only calls ``S_ISLNK`` misses
313
+ the one unprivileged local-filesystem redirection primitive an
314
+ unprivileged Windows attacker can actually create — a true Windows
315
+ symlink needs ``SeCreateSymbolicLinkPrivilege``, which an ordinary local
316
+ account does not have by default. ``st_reparse_tag`` is populated only
317
+ on Windows and is ``0``/absent everywhere else, so ``getattr(..., 0)``
318
+ is a portable, no-op check on POSIX. ``os.path.isjunction`` would be the
319
+ stdlib spelling of this, but it is 3.12+ and this package's floor is
320
+ 3.11.
321
+ """
322
+ return getattr(info, "st_reparse_tag", 0) != 0
323
+
324
+
325
+ def _refuse_degenerate_identity(identity: tuple[int, int], expanded: Path) -> None:
326
+ """Refuse a ``(0, 0)`` device/inode pair rather than silently trusting
327
+ it as a real identity.
328
+
329
+ Some filesystems — certain Windows network shares among them — report
330
+ ``st_dev``/``st_ino`` as ``0`` for every entry. An identity check built
331
+ on that value would compare ``(0, 0) == (0, 0)`` for two UNRELATED
332
+ directories and pass every time, making the whole re-verification an
333
+ inert no-op precisely where a UNC ``-o \\\\server\\share\\dl`` style
334
+ destination would otherwise rely on it most.
335
+ """
336
+ if identity == (0, 0):
337
+ raise DestinationDirectoryError(
338
+ f"Refusing to use {expanded} as a download destination: the filesystem reports a "
339
+ "degenerate (0, 0) device/inode identity for it (seen on some network shares), which "
340
+ "cannot detect a swap. Choose a destination on a filesystem that reports real identities."
341
+ )
342
+
343
+
344
+ class DestinationDirectory:
345
+ """A stable handle to one download destination directory.
346
+
347
+ Acquired once, immediately before the write, by
348
+ :func:`open_destination_directory`; the write for that download must go
349
+ through :meth:`write_atomic` on this SAME instance, never through a
350
+ fresh path-string lookup. Use as a context manager, or call
351
+ :meth:`close` explicitly once the write (success or failure) is
352
+ complete.
353
+ """
354
+
355
+ #: The path this handle was opened for — display/error text only, never
356
+ #: re-resolved for a filesystem operation once the handle exists.
357
+ display_path: Path
358
+
359
+ def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
360
+ """Write *data* into ``final_name`` directly inside this directory.
361
+
362
+ A same-directory temporary file is written, flushed, and ``fsync``'d,
363
+ then published atomically. With ``replace_existing=True`` it replaces
364
+ a prior complete entry; otherwise publication fails if *any* final
365
+ entry already exists. Any failure after temporary-file creation may
366
+ leave that mode-0600 ``.part`` entry behind. No backend deletes it
367
+ automatically because a pathname can be replaced before cleanup.
368
+ Ordinary write and publication errors name the temporary entry and ask
369
+ the operator to locate and verify it before manual cleanup.
370
+ ``KeyboardInterrupt`` and
371
+ ``SystemExit`` are re-raised unchanged; their possible residue is
372
+ documented here rather than reported while interrupting the caller.
373
+ ``final_name`` must be a bare basename (no path separators); see
374
+ :func:`_validate_leaf_name`.
375
+ """
376
+ raise NotImplementedError
377
+
378
+ def close(self) -> None:
379
+ raise NotImplementedError
380
+
381
+ def __enter__(self) -> "DestinationDirectory":
382
+ return self
383
+
384
+ def __exit__(self, *exc_info: object) -> None:
385
+ self.close()
386
+
387
+
388
+ class _DirFdDestinationDirectory(DestinationDirectory):
389
+ """POSIX ``openat()``-style handle: race-free by construction.
390
+
391
+ ``fd`` is an open, ``O_DIRECTORY`` file descriptor bound to the
392
+ destination's inode. Every operation below passes it as ``dir_fd=``
393
+ with a bare basename, so none of them ever re-walks ``display_path`` as
394
+ a string again — and, because a bare basename cannot contain a path
395
+ separator, none of them can ever land outside the directory ``fd``
396
+ identifies, by construction rather than by re-checking.
397
+ """
398
+
399
+ def __init__(self, fd: int, display_path: Path) -> None:
400
+ self.display_path = display_path
401
+ self._fd = fd
402
+ self._closed = False
403
+
404
+ def _mkstemp(self, final_name: str) -> tuple[int, str]:
405
+ last_exc: Optional[OSError] = None
406
+ for _ in range(_TEMP_NAME_ATTEMPTS):
407
+ candidate = _random_temp_name(final_name)
408
+ try:
409
+ fd = os.open(candidate, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600, dir_fd=self._fd)
410
+ except FileExistsError as exc:
411
+ last_exc = exc
412
+ continue
413
+ except FileNotFoundError as exc:
414
+ # The held directory was removed out from under this
415
+ # handle (e.g. an attacker who deleted a genuinely
416
+ # pre-existing directory after it was validated) — the
417
+ # kernel refuses to add new entries to an unlinked
418
+ # directory even though this fd still refers to it, which
419
+ # is itself a correct refusal. Give it a clear message
420
+ # rather than surfacing the bare temp filename.
421
+ raise DestinationDirectoryError(
422
+ f"Refusing to write into {self.display_path}: the destination directory no longer "
423
+ "exists. It may have been removed after it was validated."
424
+ ) from exc
425
+ return fd, candidate
426
+ raise DestinationDirectoryError(
427
+ f"Could not create a unique temporary file for {final_name!r} in {self.display_path} "
428
+ "after repeated attempts."
429
+ ) from last_exc
430
+
431
+ def _refuse_portable_alias(self, final_name: str, *, published_identity: Optional[tuple[int, int]] = None) -> None:
432
+ aliases = [
433
+ name
434
+ for name in os.listdir(self._fd)
435
+ if name != final_name and _portable_name_key(name) == _portable_name_key(final_name)
436
+ ]
437
+ if not aliases:
438
+ return
439
+ if published_identity is not None:
440
+ current = os.stat(final_name, dir_fd=self._fd, follow_symlinks=False)
441
+ if (current.st_dev, current.st_ino) != published_identity:
442
+ raise DestinationDirectoryError(
443
+ f"A portable-equivalent alias appeared after the complete download was published at "
444
+ f"{final_name!r}, and another process replaced that entry before identity verification. "
445
+ "Manual resolution is required; the client did not delete either entry."
446
+ )
447
+ # Python exposes no conditional unlink-by-inode primitive. Leave
448
+ # the proven self-published entry in place rather than reopening a
449
+ # stat-to-unlink race that could delete a replacement.
450
+ raise DestinationDirectoryError(
451
+ f"The complete download was published at {final_name!r}, but portable-equivalent alias "
452
+ f"{aliases[0]!r} appeared. Manual resolution is required; the client did not delete either entry."
453
+ )
454
+ raise DestinationDirectoryError(
455
+ f"Refusing to publish {final_name!r}: portable-equivalent alias {aliases[0]!r} exists."
456
+ )
457
+
458
+ def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
459
+ _validate_leaf_name(final_name)
460
+ if not replace_existing:
461
+ self._refuse_portable_alias(final_name)
462
+ fd, temp_name = self._mkstemp(final_name)
463
+ try:
464
+ published_identity = os.fstat(fd)
465
+ except (KeyboardInterrupt, SystemExit):
466
+ _close_fd_quietly(fd)
467
+ raise
468
+ except Exception as exc:
469
+ _close_fd_quietly(fd)
470
+ raise _retained_temp_error(self.display_path, temp_name, exc) from exc
471
+ try:
472
+ handle = os.fdopen(fd, "wb")
473
+ except (KeyboardInterrupt, SystemExit):
474
+ _close_fd_quietly(fd)
475
+ raise
476
+ except Exception as exc:
477
+ _close_fd_quietly(fd)
478
+ raise _retained_temp_error(self.display_path, temp_name, exc) from exc
479
+ published = False
480
+ existing_final_preserved = False
481
+ try:
482
+ with handle:
483
+ handle.write(data)
484
+ handle.flush()
485
+ os.fsync(handle.fileno())
486
+ if replace_existing:
487
+ os.replace(temp_name, final_name, src_dir_fd=self._fd, dst_dir_fd=self._fd)
488
+ published = True
489
+ else:
490
+ try:
491
+ _rename_noreplace(temp_name, final_name, src_dir_fd=self._fd, dst_dir_fd=self._fd)
492
+ except FileExistsError as exc:
493
+ existing_final_preserved = True
494
+ raise DestinationDirectoryError(
495
+ f"Refusing to replace existing download destination {final_name!r} in {self.display_path}. "
496
+ "The existing final entry was preserved."
497
+ ) from exc
498
+ published = True
499
+ self._refuse_portable_alias(
500
+ final_name, published_identity=(published_identity.st_dev, published_identity.st_ino)
501
+ )
502
+ except (KeyboardInterrupt, SystemExit):
503
+ raise
504
+ except Exception as exc:
505
+ if published:
506
+ raise
507
+ raise _retained_temp_error(
508
+ self.display_path, temp_name, exc, existing_final_preserved=existing_final_preserved
509
+ ) from exc
510
+
511
+ def close(self) -> None:
512
+ if not self._closed:
513
+ self._closed = True
514
+ os.close(self._fd)
515
+
516
+
517
+ class _PathIdentityDestinationDirectory(DestinationDirectory):
518
+ """Windows (no ``dir_fd``) fallback: narrows, but cannot eliminate, the
519
+ ancestor-component race. See the module docstring's WINDOWS section for
520
+ exactly what this class does and does not guarantee."""
521
+
522
+ def __init__(self, display_path: Path, identity: tuple[int, int], *, originally_symlink: bool) -> None:
523
+ self.display_path = display_path
524
+ #: ``(st_dev, st_ino)`` captured when this handle was opened —
525
+ #: re-verified before every filesystem operation below.
526
+ self._identity = identity
527
+ #: Whether *display_path* was ALREADY a symlink (to a real
528
+ #: directory) the first time this handle looked — an established
529
+ #: policy allows that, so every later check must keep following it
530
+ #: (and compare the TARGET's identity) rather than refusing outright
531
+ #: merely because the path is, as it always was, a symlink. A path
532
+ #: that started as a plain directory must never become one.
533
+ self._originally_symlink = originally_symlink
534
+ self._closed = False
535
+
536
+ def _verify_unchanged(self) -> None:
537
+ path_str = os.fspath(self.display_path)
538
+ follow = self._originally_symlink
539
+ try:
540
+ info = os.stat(path_str, follow_symlinks=follow)
541
+ except OSError as exc:
542
+ raise DestinationDirectoryError(
543
+ f"Download destination {self.display_path} became unreachable during the download: {exc}"
544
+ ) from exc
545
+ if not follow and (stat_module.S_ISLNK(info.st_mode) or _is_reparse_point(info)):
546
+ raise DestinationDirectoryError(
547
+ f"Refusing to write into {self.display_path}: it became a symlink or junction during " "the download."
548
+ )
549
+ if not stat_module.S_ISDIR(info.st_mode):
550
+ raise DestinationDirectoryError(f"Refusing to write into {self.display_path}: it is no longer a directory.")
551
+ if (info.st_dev, info.st_ino) != self._identity:
552
+ raise DestinationDirectoryError(
553
+ f"Refusing to write into {self.display_path}: it was replaced with a different "
554
+ "directory during the download."
555
+ )
556
+
557
+ def _mkstemp(self, final_name: str) -> tuple[int, Path]:
558
+ """Bounded retry on an ``O_EXCL`` temp-name collision — mirrors
559
+ :meth:`_DirFdDestinationDirectory._mkstemp`'s 64-attempt loop (and
560
+ the pre-fix ``tempfile.mkstemp``, which retried on every platform).
561
+ Without this, an uncaught ``FileExistsError`` on the ~1/2**64-odds
562
+ collision would simply fail the download outright instead of
563
+ retrying, on the one platform where this path is load-bearing.
564
+ """
565
+ last_exc: Optional[OSError] = None
566
+ for _ in range(_TEMP_NAME_ATTEMPTS):
567
+ temp_path = self.display_path / _random_temp_name(final_name)
568
+ try:
569
+ fd = os.open(os.fspath(temp_path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
570
+ except FileExistsError as exc:
571
+ last_exc = exc
572
+ continue
573
+ return fd, temp_path
574
+ raise DestinationDirectoryError(
575
+ f"Could not create a unique temporary file for {final_name!r} in {self.display_path} "
576
+ "after repeated attempts."
577
+ ) from last_exc
578
+
579
+ def _refuse_portable_alias(self, final_name: str, *, published_identity: Optional[tuple[int, int]] = None) -> None:
580
+ aliases = [
581
+ path
582
+ for path in self.display_path.iterdir()
583
+ if path.name != final_name and _portable_name_key(path.name) == _portable_name_key(final_name)
584
+ ]
585
+ if not aliases:
586
+ return
587
+ if published_identity is not None:
588
+ final_path = self.display_path / final_name
589
+ info = final_path.stat(follow_symlinks=False)
590
+ if (info.st_dev, info.st_ino) != published_identity:
591
+ raise DestinationDirectoryError(
592
+ f"A portable-equivalent alias appeared after the complete download was published at "
593
+ f"{final_name!r}, and another process replaced that entry before identity verification. "
594
+ "Manual resolution is required; the client did not delete either entry."
595
+ )
596
+ raise DestinationDirectoryError(
597
+ f"The complete download was published at {final_name!r}, but portable-equivalent alias "
598
+ f"{aliases[0].name!r} appeared. Manual resolution is required; the client did not delete either entry."
599
+ )
600
+ raise DestinationDirectoryError(f"Refusing to publish {final_name!r}: portable-equivalent alias exists.")
601
+
602
+ def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
603
+ _validate_leaf_name(final_name)
604
+ self._verify_unchanged()
605
+ if not replace_existing:
606
+ self._refuse_portable_alias(final_name)
607
+ fd, temp_path = self._mkstemp(final_name)
608
+ try:
609
+ published_identity = os.fstat(fd)
610
+ except (KeyboardInterrupt, SystemExit):
611
+ _close_fd_quietly(fd)
612
+ raise
613
+ except Exception as exc:
614
+ _close_fd_quietly(fd)
615
+ raise _retained_temp_error(self.display_path, temp_path.name, exc) from exc
616
+ final_path = self.display_path / final_name
617
+ try:
618
+ handle = os.fdopen(fd, "wb")
619
+ except (KeyboardInterrupt, SystemExit):
620
+ _close_fd_quietly(fd)
621
+ raise
622
+ except Exception as exc:
623
+ _close_fd_quietly(fd)
624
+ raise _retained_temp_error(self.display_path, temp_path.name, exc) from exc
625
+ published = False
626
+ existing_final_preserved = False
627
+ try:
628
+ with handle:
629
+ handle.write(data)
630
+ handle.flush()
631
+ os.fsync(handle.fileno())
632
+ self._verify_unchanged()
633
+ if replace_existing:
634
+ os.replace(os.fspath(temp_path), os.fspath(final_path))
635
+ published = True
636
+ else:
637
+ try:
638
+ _rename_noreplace(os.fspath(temp_path), os.fspath(final_path))
639
+ except FileExistsError as exc:
640
+ existing_final_preserved = True
641
+ raise DestinationDirectoryError(
642
+ f"Refusing to replace existing download destination {final_name!r} in {self.display_path}. "
643
+ "The existing final entry was preserved."
644
+ ) from exc
645
+ published = True
646
+ self._refuse_portable_alias(
647
+ final_name, published_identity=(published_identity.st_dev, published_identity.st_ino)
648
+ )
649
+ except (KeyboardInterrupt, SystemExit):
650
+ raise
651
+ except Exception as exc:
652
+ if published:
653
+ raise
654
+ raise _retained_temp_error(
655
+ self.display_path, temp_path.name, exc, existing_final_preserved=existing_final_preserved
656
+ ) from exc
657
+
658
+ def close(self) -> None:
659
+ self._closed = True
660
+
661
+
662
+ # ---------------------------------------------------------------------------
663
+ # Phase 1: observe (read-only, before any network I/O)
664
+ # ---------------------------------------------------------------------------
665
+
666
+
667
+ class _ComponentObservation:
668
+ """One path component's state as observed, before the fetch: its name,
669
+ and either the resolved (device, inode) identity it had (existed), or
670
+ ``None`` (did not exist — and, transitively, neither did anything below
671
+ it)."""
672
+
673
+ __slots__ = ("name", "identity")
674
+
675
+ def __init__(self, name: str, identity: Optional[tuple[int, int]]) -> None:
676
+ self.name = name
677
+ self.identity = identity
678
+
679
+
680
+ class DestinationObservation:
681
+ """Opaque snapshot of a destination directory's pre-fetch state.
682
+
683
+ Produced by :func:`observe_destination`, consumed by
684
+ :func:`open_destination_directory`. Never constructed directly, and
685
+ never partially reused across two different destinations — call
686
+ :func:`observe_destination` again for a different path.
687
+
688
+ On POSIX, this may hold an OPEN file descriptor for the deepest
689
+ existing path component (``held_fd``) for the entire gap between
690
+ observation and open — see the module docstring for why a held
691
+ descriptor, not a recorded identity value, is what closes the race for
692
+ that prefix. Callers MUST call :meth:`close` (or use this as a context
693
+ manager) on every path, including a digest mismatch or a failed fetch,
694
+ so that descriptor is never leaked. :func:`open_destination_directory`
695
+ takes ownership of it on success — a redundant :meth:`close` afterward
696
+ is always safe.
697
+ """
698
+
699
+ def __init__(
700
+ self,
701
+ expanded: Path,
702
+ *,
703
+ is_absolute: bool,
704
+ held_fd: Optional[int] = None,
705
+ absent_names: Optional[list[str]] = None,
706
+ components: Optional[list[_ComponentObservation]] = None,
707
+ existed: bool = True,
708
+ identity: Optional[tuple[int, int]] = None,
709
+ was_symlink: bool = False,
710
+ ) -> None:
711
+ self.expanded = expanded
712
+ self.is_absolute = is_absolute
713
+ #: POSIX only: an open fd for the deepest EXISTING component, or
714
+ #: ``None`` if nothing below the anchor existed (a relative
715
+ #: destination with no existing prefix at all — the anchor is then
716
+ #: the process's own current working directory, itself immune to a
717
+ #: path-string race; see :func:`_open_posix`). Ownership transfers
718
+ #: to the :class:`DestinationDirectory` :func:`open_destination_directory`
719
+ #: returns on success.
720
+ self.held_fd = held_fd
721
+ #: POSIX only: the ordered suffix of component names that did NOT
722
+ #: exist at observation time and must be created (each guarded by
723
+ #: ``O_NOFOLLOW`` at open time).
724
+ self.absent_names = absent_names
725
+ #: The full per-component chain, Windows-fallback only (``None`` on
726
+ #: POSIX, which uses ``held_fd``/``absent_names`` instead — see the
727
+ #: module docstring's WINDOWS section for why the fallback cannot
728
+ #: use a held handle the same way).
729
+ self.components = components
730
+ #: Convenience mirror of the FINAL component's own state — fallback only.
731
+ self.existed = existed
732
+ self.identity = identity
733
+ self.was_symlink = was_symlink
734
+ self._closed = False
735
+ self._consumed = False
736
+
737
+ def _take_for_open(self) -> None:
738
+ """Claim this observation for exactly one
739
+ :func:`open_destination_directory` call, refusing reuse.
740
+
741
+ A containment primitive whose misuse mode is "silently create and
742
+ write in a DIFFERENT directory" is the wrong direction to fail.
743
+ Without this guard, a second open on an already-spent observation
744
+ (``held_fd`` already handed off, ``None``) falls through to an
745
+ unqualified, ``dir_fd=None`` — i.e. CWD-relative — walk on POSIX,
746
+ landing bytes wherever the process happens to be running rather
747
+ than the validated destination; opening after :meth:`close` raises
748
+ a raw, non-``OSError`` exception (an ``AssertionError``, or under
749
+ ``python -O`` a ``TypeError`` from operating on a ``None`` fd) that
750
+ the CLI's ``except (OSError, ValueError)`` does not catch. Refusing
751
+ outright, the same way for both misuse shapes, is strictly safer
752
+ than either.
753
+ """
754
+ if self._consumed:
755
+ raise DestinationDirectoryError(
756
+ f"Refusing to open {self.expanded}: this observation was already used for a previous "
757
+ "open_destination_directory() call, or was closed first. Call observe_destination() "
758
+ "again to snapshot the destination and attempt a new open."
759
+ )
760
+ self._consumed = True
761
+
762
+ def is_existing_directory(self) -> bool:
763
+ """Whether the selected directory existed when this observation began."""
764
+ if self.absent_names is not None:
765
+ return not self.absent_names
766
+ return self.existed
767
+
768
+ def matches_directory(self, directory: Path) -> bool:
769
+ """Whether *directory* is exactly the path this observation owns."""
770
+ return self.expanded == Path(os.path.expanduser(os.fspath(directory)))
771
+
772
+ def contained_destination(self, final_name: str) -> Path:
773
+ """Return a final path that passed this observation's containment check.
774
+
775
+ The later open consumes this same observation, so a selected directory
776
+ swapped after this check cannot redirect the write. A swap visible to
777
+ the path-resolution check instead causes a refusal.
778
+ """
779
+ _validate_leaf_name(final_name)
780
+ destination = self.expanded / final_name
781
+ try:
782
+ resolved_root = self.expanded.resolve(strict=False)
783
+ resolved_destination = destination.resolve(strict=False)
784
+ resolved_destination.relative_to(resolved_root)
785
+ except (OSError, RuntimeError, ValueError) as exc:
786
+ raise DestinationDirectoryError(
787
+ f"Could not validate {destination} beneath selected directory {self.expanded}: {exc}"
788
+ ) from exc
789
+ if resolved_destination == resolved_root:
790
+ raise DestinationDirectoryError(
791
+ f"Refusing to write {final_name!r}: it resolves to the selected directory itself."
792
+ )
793
+ return destination
794
+
795
+ def close(self) -> None:
796
+ """Release the held descriptor, if any, and mark this observation
797
+ spent — a later :func:`open_destination_directory` call on it must
798
+ refuse outright (see :meth:`_take_for_open`) rather than silently
799
+ falling through to an unqualified, CWD-relative walk. Idempotent
800
+ and safe to call after ownership has already transferred to a
801
+ :class:`DestinationDirectory` (``held_fd`` is ``None`` by then)."""
802
+ if not self._closed:
803
+ self._closed = True
804
+ self._consumed = True
805
+ if self.held_fd is not None:
806
+ os.close(self.held_fd)
807
+ self.held_fd = None
808
+
809
+ def __enter__(self) -> "DestinationObservation":
810
+ return self
811
+
812
+ def __exit__(self, *exc_info: object) -> None:
813
+ self.close()
814
+
815
+
816
+ def _observe_posix(directory: Path) -> DestinationObservation:
817
+ expanded = Path(os.path.expanduser(os.fspath(directory)))
818
+ is_absolute = expanded.is_absolute()
819
+ parts = expanded.parts
820
+ remaining = parts[1:] if is_absolute else parts
821
+ absent_names: list[str] = []
822
+ #: The deepest component confirmed to exist so far — ``None`` means
823
+ #: "relative to the process's own current working directory" (either
824
+ #: because *expanded* is relative and nothing in it exists yet, or
825
+ #: because the walk has not opened anything below the anchor at all).
826
+ current_fd: Optional[int] = None
827
+ stop = False
828
+ try:
829
+ if is_absolute:
830
+ try:
831
+ current_fd = os.open(parts[0], os.O_DIRECTORY | os.O_RDONLY)
832
+ except OSError as exc:
833
+ raise DestinationDirectoryError(f"Could not observe download destination {expanded}: {exc}") from exc
834
+ for name in remaining:
835
+ if stop:
836
+ absent_names.append(name)
837
+ continue
838
+ try:
839
+ next_fd = os.open(name, os.O_DIRECTORY | os.O_RDONLY, dir_fd=current_fd)
840
+ except FileNotFoundError:
841
+ absent_names.append(name)
842
+ stop = True
843
+ continue
844
+ except NotADirectoryError as exc:
845
+ raise DestinationDirectoryError(f"{expanded}: {name!r} exists and is not a directory.") from exc
846
+ except OSError as exc:
847
+ raise DestinationDirectoryError(f"Could not observe download destination {expanded}: {exc}") from exc
848
+ # `next_fd` becomes the new deepest-known-existing component —
849
+ # release the fd it superseded (never the one we are about to
850
+ # return) BEFORE advancing, so a later exception in this loop
851
+ # closes at most one live fd via the `except` below, never two.
852
+ if current_fd is not None:
853
+ os.close(current_fd)
854
+ current_fd = next_fd
855
+ except BaseException:
856
+ if current_fd is not None:
857
+ os.close(current_fd)
858
+ raise
859
+ # `current_fd` is now the deepest EXISTING component's fd (or the
860
+ # anchor's, or None for "nothing exists, relative to cwd") — HELD open,
861
+ # deliberately, not closed: this is what pins its inode against reuse
862
+ # for the entire gap until `open_destination_directory` runs. See the
863
+ # module docstring.
864
+ was_symlink = False
865
+ if not absent_names:
866
+ try:
867
+ was_symlink = stat_module.S_ISLNK(os.lstat(os.fspath(expanded)).st_mode)
868
+ except OSError as exc:
869
+ if current_fd is not None:
870
+ os.close(current_fd)
871
+ raise DestinationDirectoryError(f"Could not observe download destination {expanded}: {exc}") from exc
872
+ return DestinationObservation(
873
+ expanded,
874
+ is_absolute=is_absolute,
875
+ held_fd=current_fd,
876
+ absent_names=absent_names,
877
+ was_symlink=was_symlink,
878
+ )
879
+
880
+
881
+ def _observe_fallback(directory: Path) -> DestinationObservation:
882
+ """The Windows-fallback observation: no ``dir_fd`` to hold or chain, so
883
+ each component is checked by ``os.stat``ing the ACCUMULATED path
884
+ string built up so far, and its (device, inode) VALUE is recorded —
885
+ weaker than the POSIX walk (which holds the deepest existing
886
+ component's descriptor open instead of trusting a recorded value; see
887
+ the module docstring for why a value can be fooled by inode reuse) but
888
+ still a real per-component snapshot, closing the same ancestor gap the
889
+ POSIX walk closes, just not to zero width — see the module docstring's
890
+ WINDOWS section for exactly what residual window this leaves.
891
+ """
892
+ expanded = Path(os.path.expanduser(os.fspath(directory)))
893
+ is_absolute = expanded.is_absolute()
894
+ parts = expanded.parts
895
+ remaining = parts[1:] if is_absolute else parts
896
+ components: list[_ComponentObservation] = []
897
+ accumulated = Path(parts[0]) if is_absolute else Path(".")
898
+ stop = False
899
+ for name in remaining:
900
+ accumulated = accumulated / name
901
+ if stop:
902
+ components.append(_ComponentObservation(name, None))
903
+ continue
904
+ try:
905
+ info = os.stat(os.fspath(accumulated), follow_symlinks=True)
906
+ except FileNotFoundError:
907
+ components.append(_ComponentObservation(name, None))
908
+ stop = True
909
+ continue
910
+ except OSError as exc:
911
+ raise DestinationDirectoryError(f"Could not observe download destination {expanded}: {exc}") from exc
912
+ if not stat_module.S_ISDIR(info.st_mode):
913
+ raise DestinationDirectoryError(f"{expanded}: {name!r} exists and is not a directory.")
914
+ identity = (info.st_dev, info.st_ino)
915
+ _refuse_degenerate_identity(identity, expanded)
916
+ components.append(_ComponentObservation(name, identity))
917
+
918
+ leaf = components[-1] if components else None
919
+ existed = leaf.identity is not None if leaf is not None else True
920
+ leaf_identity = leaf.identity if leaf is not None else None
921
+ was_symlink = False
922
+ if existed:
923
+ try:
924
+ leaf_lstat = os.lstat(os.fspath(expanded))
925
+ except OSError as exc:
926
+ raise DestinationDirectoryError(f"Could not observe download destination {expanded}: {exc}") from exc
927
+ if _is_reparse_point(leaf_lstat):
928
+ raise DestinationDirectoryError(
929
+ f"Refusing to use {expanded} as a download destination: it is a directory junction, "
930
+ "which this module does not treat as a followable selected-directory symlink."
931
+ )
932
+ was_symlink = stat_module.S_ISLNK(leaf_lstat.st_mode)
933
+ return DestinationObservation(
934
+ expanded,
935
+ is_absolute=is_absolute,
936
+ components=components,
937
+ existed=existed,
938
+ identity=leaf_identity,
939
+ was_symlink=was_symlink,
940
+ )
941
+
942
+
943
+ def observe_destination(directory: Path) -> DestinationObservation:
944
+ """Snapshot *directory*'s current state for a download about to begin.
945
+
946
+ Call this FIRST — before any network I/O for the download, including
947
+ resolving a presigned URL — and hold the result until bytes are ready
948
+ to write, then pass it to :func:`open_destination_directory`. This call
949
+ is read-only: it CREATES nothing, so it adds no new filesystem footprint
950
+ to a fetch that ultimately fails (a digest mismatch, a network error) —
951
+ that guarantee is unchanged from before this module existed. On POSIX
952
+ it may hold an open file descriptor for the deepest existing path
953
+ component until the returned :class:`DestinationObservation` is closed
954
+ (see its docstring) — a process-local resource, not a footprint on
955
+ disk, but the caller MUST close it on every path, success or failure.
956
+
957
+ Relative paths and ``~`` expansion are both honored, matching every
958
+ existing ``-o PATH``/implicit-cwd download destination.
959
+ """
960
+ if SUPPORTS_DIR_FD:
961
+ return _observe_posix(directory)
962
+ return _observe_fallback(directory)
963
+
964
+
965
+ # ---------------------------------------------------------------------------
966
+ # Phase 2: open (immediately before the write, using the observation)
967
+ # ---------------------------------------------------------------------------
968
+
969
+
970
+ def _create_absent_component(parent_fd: Optional[int], name: str, expanded: Path) -> int:
971
+ """Create ``name`` directly inside ``parent_fd`` (``None`` means
972
+ "relative to the process's own current working directory", itself
973
+ immune to a path-string race), never following anything already there.
974
+
975
+ A bare ``os.mkdir`` is attempted first (fails outright — symlink or
976
+ not — if anything already occupies the name); whatever is then opened
977
+ is opened ``O_NOFOLLOW``. Since the observation already proved nothing
978
+ was here when the operation began, anything other than a plain
979
+ directory now is provably a race and is refused rather than followed.
980
+
981
+ There is no "observed PRESENT" counterpart to this function on the
982
+ ``dir_fd`` path: a component that existed at observation time is never
983
+ re-walked here at all — :func:`_open_posix` continues directly from the
984
+ file descriptor :func:`observe_destination` kept open for it. See the
985
+ module docstring for why a held descriptor, not a re-checked identity
986
+ value, is what closes the race for that prefix.
987
+ """
988
+ try:
989
+ os.mkdir(name, dir_fd=parent_fd)
990
+ except FileExistsError:
991
+ pass # benign concurrent create OR an attacker's plant -- the NOFOLLOW open below decides
992
+ except OSError as exc:
993
+ raise DestinationDirectoryError(f"Could not create download destination {expanded}: {exc}") from exc
994
+ try:
995
+ return os.open(name, os.O_DIRECTORY | os.O_RDONLY | os.O_NOFOLLOW, dir_fd=parent_fd)
996
+ except OSError as exc:
997
+ raise DestinationDirectoryError(
998
+ f"Refusing to use {expanded} as a download destination: {name!r} was replaced with a "
999
+ "symlink (or another non-directory) after it was validated as missing. This can happen "
1000
+ "when another process races the same path — pick a different -o PATH, or remove the "
1001
+ "conflicting entry and retry."
1002
+ ) from exc
1003
+
1004
+
1005
+ def _open_posix(observation: DestinationObservation) -> _DirFdDestinationDirectory:
1006
+ expanded = observation.expanded
1007
+ assert observation.absent_names is not None # POSIX observations always populate this
1008
+
1009
+ # Take ownership of the held descriptor immediately: from this point,
1010
+ # `observation.close()` must not touch it (it is about to become either
1011
+ # this walk's own `current_fd`, closed on failure right here, or the
1012
+ # returned handle's fd) — a caller that calls `observation.close()`
1013
+ # again afterward, in an outer `finally`, must see a safe no-op.
1014
+ current_fd = observation.held_fd
1015
+ observation.held_fd = None
1016
+ try:
1017
+ if current_fd is not None and not observation.absent_names:
1018
+ try:
1019
+ leaf_lstat = os.lstat(os.fspath(expanded))
1020
+ leaf_stat = os.stat(os.fspath(expanded), follow_symlinks=True)
1021
+ except OSError as exc:
1022
+ raise DestinationDirectoryError(
1023
+ f"Download destination {expanded} became unreachable during the download: {exc}"
1024
+ ) from exc
1025
+ if not observation.was_symlink and stat_module.S_ISLNK(leaf_lstat.st_mode):
1026
+ raise DestinationDirectoryError(
1027
+ f"Refusing to write into {expanded}: it became a symlink during the download."
1028
+ )
1029
+ held_stat = os.fstat(current_fd)
1030
+ if (leaf_stat.st_dev, leaf_stat.st_ino) != (held_stat.st_dev, held_stat.st_ino):
1031
+ raise DestinationDirectoryError(
1032
+ f"Refusing to write into {expanded}: it was replaced with a different directory during the download."
1033
+ )
1034
+ if current_fd is None and not observation.is_absolute:
1035
+ # Nothing existed below the anchor, and the anchor itself is
1036
+ # the process's own current working directory (an absolute
1037
+ # destination always has at least the root's own fd held —
1038
+ # see `_observe_posix`). `dir_fd=None` below already means
1039
+ # "resolve relative to cwd" to the kernel, which is bound to
1040
+ # this process's `chdir()` state, not a re-walked path string
1041
+ # — so this needs no fd of its own to stay race-free; opening
1042
+ # one explicitly is just for a uniform, always-holds-an-fd
1043
+ # `_DirFdDestinationDirectory` instead of a special `None` case
1044
+ # threaded through every one of its methods.
1045
+ current_fd = os.open(".", os.O_DIRECTORY | os.O_RDONLY)
1046
+ for name in observation.absent_names:
1047
+ next_fd = _create_absent_component(current_fd, name, expanded)
1048
+ if current_fd is not None:
1049
+ os.close(current_fd)
1050
+ current_fd = next_fd
1051
+ except BaseException:
1052
+ if current_fd is not None:
1053
+ os.close(current_fd)
1054
+ raise
1055
+ assert current_fd is not None
1056
+ return _DirFdDestinationDirectory(current_fd, expanded)
1057
+
1058
+
1059
+ def _open_fallback_component(accumulated: Path, comp: _ComponentObservation, expanded: Path) -> None:
1060
+ """Verify or create ONE component of the fallback's path-string walk,
1061
+ at *accumulated* (the full path up to and including this component).
1062
+
1063
+ Unlike the POSIX path (:func:`_open_posix`, :func:`_create_absent_component`),
1064
+ there is no ``dir_fd`` to hold: each check here is its own separate
1065
+ ``os.stat``/``os.mkdir`` pair against an ACCUMULATED path string. A
1066
+ swap of THIS component after ITS OWN check has already run redirects
1067
+ every SUBSEQUENT component's ``mkdir``/``stat`` for the REMAINDER of
1068
+ the walk — not merely "the next syscall on this same component" — since
1069
+ every later component's accumulated path is built by extending this
1070
+ one's string. See the module docstring's WINDOWS section.
1071
+ """
1072
+ name = comp.name
1073
+ if comp.identity is None:
1074
+ try:
1075
+ os.mkdir(os.fspath(accumulated))
1076
+ except FileExistsError:
1077
+ pass # benign concurrent create OR an attacker's plant -- the check below decides
1078
+ except OSError as exc:
1079
+ raise DestinationDirectoryError(f"Could not create download destination {expanded}: {exc}") from exc
1080
+ try:
1081
+ info = os.stat(os.fspath(accumulated), follow_symlinks=False)
1082
+ except OSError as exc:
1083
+ raise DestinationDirectoryError(f"Could not open download destination {expanded}: {exc}") from exc
1084
+ # Observed ABSENT: anything other than a plain, non-reparse
1085
+ # directory here now is provably a race, since the observation
1086
+ # already proved nothing was here when the operation began.
1087
+ if stat_module.S_ISLNK(info.st_mode) or _is_reparse_point(info):
1088
+ raise DestinationDirectoryError(
1089
+ f"Refusing to use {expanded} as a download destination: {name!r} was replaced with a "
1090
+ "symlink or junction after it was validated as missing."
1091
+ )
1092
+ if not stat_module.S_ISDIR(info.st_mode):
1093
+ raise DestinationDirectoryError(f"{expanded}: {name!r} exists and is not a directory.")
1094
+ return
1095
+
1096
+ try:
1097
+ info = os.stat(os.fspath(accumulated), follow_symlinks=True)
1098
+ except OSError as exc:
1099
+ raise DestinationDirectoryError(
1100
+ f"Refusing to use {expanded} as a download destination: {name!r} changed since it was "
1101
+ f"validated ({exc})."
1102
+ ) from exc
1103
+ if not stat_module.S_ISDIR(info.st_mode):
1104
+ raise DestinationDirectoryError(
1105
+ f"Refusing to use {expanded} as a download destination: {name!r} is not a directory."
1106
+ )
1107
+ identity = (info.st_dev, info.st_ino)
1108
+ if identity != comp.identity:
1109
+ raise DestinationDirectoryError(
1110
+ f"Refusing to use {expanded} as a download destination: {name!r} was replaced with a "
1111
+ "different directory after it was validated."
1112
+ )
1113
+
1114
+
1115
+ def _open_fallback(observation: DestinationObservation) -> _PathIdentityDestinationDirectory:
1116
+ expanded = observation.expanded
1117
+ assert observation.components is not None
1118
+ accumulated = Path(expanded.parts[0]) if observation.is_absolute else Path(".")
1119
+ for comp in observation.components:
1120
+ accumulated = accumulated / comp.name
1121
+ _open_fallback_component(accumulated, comp, expanded)
1122
+
1123
+ # Every component, ancestors included, is now verified/created and
1124
+ # matches the pre-fetch observation. Capture the FINAL entry's current
1125
+ # identity (and whether it is itself a symlink, which decides how
1126
+ # `_PathIdentityDestinationDirectory`'s ONGOING re-checks behave) to
1127
+ # bind the returned handle to — this is the earliest point after the
1128
+ # fetch this call can check it, mirroring what the POSIX fd already
1129
+ # identifies at this same point in the walk.
1130
+ try:
1131
+ leaf_lstat = os.stat(os.fspath(expanded), follow_symlinks=False)
1132
+ except OSError as exc:
1133
+ raise DestinationDirectoryError(f"Could not open download destination {expanded}: {exc}") from exc
1134
+ originally_symlink = stat_module.S_ISLNK(leaf_lstat.st_mode)
1135
+ try:
1136
+ info = os.stat(os.fspath(expanded), follow_symlinks=True)
1137
+ except OSError as exc:
1138
+ raise DestinationDirectoryError(f"Could not open download destination {expanded}: {exc}") from exc
1139
+ if not stat_module.S_ISDIR(info.st_mode):
1140
+ raise DestinationDirectoryError(f"Refusing to use {expanded} as a download destination: it is not a directory.")
1141
+ identity = (info.st_dev, info.st_ino)
1142
+ _refuse_degenerate_identity(identity, expanded)
1143
+ return _PathIdentityDestinationDirectory(expanded, identity, originally_symlink=originally_symlink)
1144
+
1145
+
1146
+ def open_destination_directory(observation: DestinationObservation) -> DestinationDirectory:
1147
+ """Open the destination :func:`observe_destination` snapshotted,
1148
+ honoring exactly what it recorded — there is no separate ``directory``
1149
+ argument to (mis)match against ``observation`` here: the observation
1150
+ already carries the expanded path (``observation.expanded``), and a
1151
+ second, independently-suppliable path parameter would only create a
1152
+ caller-bug opportunity for the two to diverge with no attacker
1153
+ involved. Call this immediately before the write it will hold — once
1154
+ bytes are ready to land, with no further network I/O between this call
1155
+ and :meth:`DestinationDirectory.write_atomic`.
1156
+
1157
+ A pre-existing, empty (or non-existing) real directory opens exactly as
1158
+ before; a missing one — including every missing ancestor a nested
1159
+ ``-o a/b/c`` needs — is created; see the module docstring for exactly
1160
+ which race this closes on each platform.
1161
+
1162
+ Raises :class:`DestinationDirectoryError` outright if ``observation``
1163
+ was already used for a prior call, or was already closed — this
1164
+ function may be called AT MOST ONCE per :func:`observe_destination`
1165
+ result; see :meth:`DestinationObservation._take_for_open`.
1166
+
1167
+ Ownership of any descriptor ``observation`` was holding transfers to
1168
+ the returned handle on SUCCESS. On a RAISED exception, this call (via
1169
+ ``_open_posix``/``_open_fallback``) has already closed whatever it was
1170
+ holding at the point of failure — it does not leak. Callers must still
1171
+ call ``observation.close()`` afterward in a ``finally`` regardless,
1172
+ because a failure can also occur BEFORE this function is ever called
1173
+ (a digest mismatch, a failed fetch) — that is the path this function
1174
+ cannot clean up after, since it never ran; ``observation.close()`` is
1175
+ what covers it, and is always a safe no-op once ownership has already
1176
+ transferred here.
1177
+ """
1178
+ observation._take_for_open()
1179
+ if SUPPORTS_DIR_FD:
1180
+ return _open_posix(observation)
1181
+ return _open_fallback(observation)