simulo 0.26.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- simulo/__init__.py +433 -0
- simulo/_client/__init__.py +6 -0
- simulo/_client/_entrypoint.py +313 -0
- simulo/_client/_mounts.py +25 -0
- simulo/_client/_runner.py +186 -0
- simulo/_client/_secure_downloads.py +1181 -0
- simulo/_client/app.py +1308 -0
- simulo/_client/asset.py +331 -0
- simulo/_client/asset_api.py +517 -0
- simulo/_client/asset_package.py +1103 -0
- simulo/_client/asset_pins.py +187 -0
- simulo/_client/builtin_aliases.py +107 -0
- simulo/_client/bundle.py +254 -0
- simulo/_client/cancel_api.py +104 -0
- simulo/_client/cli.py +9063 -0
- simulo/_client/config.py +186 -0
- simulo/_client/credentials.py +210 -0
- simulo/_client/discovery.py +214 -0
- simulo/_client/export_api.py +212 -0
- simulo/_client/export_bundle.py +296 -0
- simulo/_client/facades.py +581 -0
- simulo/_client/http.py +414 -0
- simulo/_client/identity_api.py +117 -0
- simulo/_client/install_samples.py +267 -0
- simulo/_client/jobs_api.py +224 -0
- simulo/_client/learning.py +393 -0
- simulo/_client/login.py +319 -0
- simulo/_client/mode.py +29 -0
- simulo/_client/outputs.py +116 -0
- simulo/_client/packaging.py +445 -0
- simulo/_client/preflight_api.py +186 -0
- simulo/_client/preflight_render.py +200 -0
- simulo/_client/registry.py +98 -0
- simulo/_client/runtime.py +185 -0
- simulo/_client/runtime_display.py +90 -0
- simulo/_client/seed_ref.py +76 -0
- simulo/_client/stub.py +41 -0
- simulo/_client/submit_api.py +1057 -0
- simulo/_client/templates/__init__.py +21 -0
- simulo/_client/templates/inference/app.py.tmpl +316 -0
- simulo/_client/templates/inference/simuloignore.tmpl +30 -0
- simulo/_client/templates/scenario/app.py.tmpl +93 -0
- simulo/_client/templates/scenario/simuloignore.tmpl +27 -0
- simulo/_client/templates/training/app.py.tmpl +235 -0
- simulo/_client/templates/training/simuloignore.tmpl +29 -0
- simulo/_client/view_fragment.py +21 -0
- simulo/_client/view_session_api.py +122 -0
- simulo/_client/volume.py +71 -0
- simulo/callbacks.py +274 -0
- simulo/py.typed +0 -0
- simulo-0.26.0.dist-info/METADATA +130 -0
- simulo-0.26.0.dist-info/RECORD +55 -0
- simulo-0.26.0.dist-info/WHEEL +5 -0
- simulo-0.26.0.dist-info/entry_points.txt +2 -0
- simulo-0.26.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1181 @@
|
|
|
1
|
+
"""Race-resistant destination-directory handling for CLI downloads.
|
|
2
|
+
|
|
3
|
+
THE RACE THIS CLOSES. The bulk download path validates its assembled
|
|
4
|
+
filename and the selected destination directory as PATH STRINGS
|
|
5
|
+
(``_contained_bulk_download_destination`` in ``cli.py``: resolve, then
|
|
6
|
+
``relative_to``) before any bytes are fetched. The actual write — inside
|
|
7
|
+
this client's ``download_model``/``download_output``/``download_recording``
|
|
8
|
+
— re-derives the SAME destination from its path string again, after a
|
|
9
|
+
network fetch: an ``exist_ok=True`` ``mkdir`` (which silently ACCEPTS a
|
|
10
|
+
symlink planted at a previously-missing path, so long as it resolves to a
|
|
11
|
+
directory), a SEPARATE ``tempfile.mkstemp(dir=...)`` call, then a SEPARATE
|
|
12
|
+
``os.replace`` call. A local attacker who can create files in whatever
|
|
13
|
+
directory CONTAINS the selected destination — or in any ANCESTOR of it a
|
|
14
|
+
nested ``-o a/b/c`` would create — can replace a previously-missing
|
|
15
|
+
directory with a symlink at any point before the eventual write and
|
|
16
|
+
redirect the downloaded bytes outside the approved destination. This was
|
|
17
|
+
named, and explicitly deferred, by the security review of the change that
|
|
18
|
+
landed the response-ID, filename, normalization, and existing-symlink-escape
|
|
19
|
+
containment this module extends.
|
|
20
|
+
|
|
21
|
+
THE FIX is two-phase, and the split matters: **observe, then open.**
|
|
22
|
+
|
|
23
|
+
:func:`observe_destination` runs FIRST — before any network I/O for the
|
|
24
|
+
download begins — and walks the ENTIRE destination path, component by
|
|
25
|
+
component, from a trusted anchor (filesystem root for an absolute path, the
|
|
26
|
+
process's current working directory for a relative one) down to the leaf.
|
|
27
|
+
For the PREFIX of components that already exist, it does not merely record
|
|
28
|
+
their (device, inode) VALUES — an inode number is a value a filesystem
|
|
29
|
+
allocator can and does reuse (measured on ext4: a directory removed with
|
|
30
|
+
``rmdir`` frees its inode for the very next ``mkdir`` in the same parent, on
|
|
31
|
+
the first attempt), so a value comparison taken later can be fooled by an
|
|
32
|
+
attacker who deletes the observed directory and lets a NEW directory they
|
|
33
|
+
control claim the same number before the eventual check runs. Instead, the
|
|
34
|
+
DEEPEST existing component's file descriptor is kept OPEN — held inside the
|
|
35
|
+
returned :class:`DestinationObservation` — which pins its inode: the kernel
|
|
36
|
+
will not hand that inode number to a new allocation while any process holds
|
|
37
|
+
it open, so the value-reuse attack has no purchase on it. Every component
|
|
38
|
+
AFTER that point does not exist yet; only its NAME is recorded, nothing
|
|
39
|
+
about its contents is trusted. This call is read-only for anything it does
|
|
40
|
+
not already hold open, so a subsequent digest mismatch or failed fetch
|
|
41
|
+
still leaves zero NEW filesystem footprint, exactly as before.
|
|
42
|
+
|
|
43
|
+
:func:`open_destination_directory` runs SECOND — immediately before the
|
|
44
|
+
atomic write, once bytes are ready to land — and takes that observation as
|
|
45
|
+
a REQUIRED argument. On POSIX it does NOT re-walk the existing prefix at
|
|
46
|
+
all: it takes OVER the held file descriptor directly (ownership transfers
|
|
47
|
+
to the returned handle) and continues the walk, ``dir_fd``-chained, only
|
|
48
|
+
for the components the observation recorded as absent — creating each with
|
|
49
|
+
a bare ``os.mkdir`` (which fails outright, symlink or not, if anything
|
|
50
|
+
already occupies the name) and then opening it ``O_NOFOLLOW``, so anything
|
|
51
|
+
other than the plain directory just created is provably a race (nothing was
|
|
52
|
+
there when the operation began) and is refused rather than followed.
|
|
53
|
+
|
|
54
|
+
Because the existing prefix is never re-derived from a path string —
|
|
55
|
+
``open_destination_directory`` continues from the SAME open descriptor
|
|
56
|
+
``observe_destination`` acquired, never a fresh lookup of the same name —
|
|
57
|
+
and every absent component is only ever created under ``O_NOFOLLOW``,
|
|
58
|
+
nothing that happens during the network fetch can redirect the eventual
|
|
59
|
+
write, for EVERY component of the destination this module creates or
|
|
60
|
+
holds, not only the final one.
|
|
61
|
+
|
|
62
|
+
MEASURED RESIDUAL. The gap between ``observe_destination`` and
|
|
63
|
+
``open_destination_directory`` is NOT small: callers call the first before
|
|
64
|
+
even resolving a presigned URL and the second only after the network fetch
|
|
65
|
+
(and any digest verification) completes — the gap IS the fetch's entire,
|
|
66
|
+
possibly long, duration, deliberately, because a digest mismatch or a
|
|
67
|
+
failed fetch must still leave zero new filesystem footprint. The invariant
|
|
68
|
+
this module provides does not depend on that gap's width. On POSIX, the
|
|
69
|
+
existing prefix is bound to a held file descriptor from the moment it is
|
|
70
|
+
observed, so nothing that happens to its NAME during the gap — deletion,
|
|
71
|
+
inode reuse at the freed number, a symlink planted at the freed name — can
|
|
72
|
+
change what the returned handle operates on, because the handle never
|
|
73
|
+
looks that name up again; and every component recorded absent is only ever
|
|
74
|
+
created under ``O_NOFOLLOW`` at open time, so a plant during the gap is
|
|
75
|
+
refused there instead. What is genuinely out of scope: the trusted anchor
|
|
76
|
+
itself (filesystem root, or the process's own current working directory),
|
|
77
|
+
already outside this module's and the CLI's threat model, and the CALLER's
|
|
78
|
+
own responsibility to pass ``observe_destination`` and
|
|
79
|
+
``open_destination_directory`` the intended destination in the first place
|
|
80
|
+
(a caller bug there is not an attacker capability this module can guard).
|
|
81
|
+
|
|
82
|
+
WINDOWS. CPython's ``os`` module does not expose ``dir_fd``-relative
|
|
83
|
+
operations on Windows for any primitive this module needs — ``os.open``,
|
|
84
|
+
``os.mkdir``, and ``os.replace`` are all documented "Availability: Unix" for
|
|
85
|
+
their ``dir_fd``/``*_dir_fd`` parameters, and ``os.supports_dir_fd`` is
|
|
86
|
+
empty there — so neither the held-descriptor technique nor the
|
|
87
|
+
``dir_fd``-chained walk is available.
|
|
88
|
+
``_PathIdentityDestinationDirectory`` still applies a per-component
|
|
89
|
+
create-or-verify treatment to every component, ancestors included, but each
|
|
90
|
+
check is its own separate ``os.stat``/``os.mkdir`` call against an
|
|
91
|
+
ACCUMULATED path string, never a held handle — so a swap of any EARLIER
|
|
92
|
+
component, at any point after that component's own check already ran,
|
|
93
|
+
redirects every SUBSEQUENT component's ``mkdir``/``stat`` for the
|
|
94
|
+
REMAINDER of that walk, not merely "the next syscall on that same
|
|
95
|
+
component". This is a real, disclosed, non-zero-width gap the POSIX path
|
|
96
|
+
does not have. Windows also has a primitive Linux/macOS do not: DIRECTORY
|
|
97
|
+
JUNCTIONS, created via ``mklink /J`` with NO elevated privilege (unlike a
|
|
98
|
+
true Windows symlink, which needs ``SeCreateSymbolicLinkPrivilege``) and
|
|
99
|
+
NOT detected by ``stat.S_ISLNK``. This module checks ``st_reparse_tag``
|
|
100
|
+
explicitly wherever it checks for a symlink on the fallback path, so the
|
|
101
|
+
unprivileged primitive an attacker can actually use is the one being
|
|
102
|
+
guarded, not only the privileged one they usually cannot. One user-visible
|
|
103
|
+
consequence: a PRE-EXISTING junction as the selected destination is now
|
|
104
|
+
refused outright, the same as a freshly-planted one, rather than silently
|
|
105
|
+
followed — junctions are the ordinary UNPRIVILEGED way Windows users link
|
|
106
|
+
directories day to day, so a ``-o`` target that is a junction, which
|
|
107
|
+
worked before this module existed, now gets a hard error where it did not
|
|
108
|
+
before.
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
from __future__ import annotations
|
|
112
|
+
|
|
113
|
+
import ctypes
|
|
114
|
+
import errno
|
|
115
|
+
import os
|
|
116
|
+
import secrets
|
|
117
|
+
import stat as stat_module
|
|
118
|
+
import sys
|
|
119
|
+
import unicodedata
|
|
120
|
+
from pathlib import Path
|
|
121
|
+
from typing import Optional
|
|
122
|
+
|
|
123
|
+
__all__ = [
|
|
124
|
+
"DestinationDirectory",
|
|
125
|
+
"DestinationDirectoryError",
|
|
126
|
+
"DestinationObservation",
|
|
127
|
+
"SUPPORTS_DIR_FD",
|
|
128
|
+
"observe_destination",
|
|
129
|
+
"open_destination_directory",
|
|
130
|
+
]
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class DestinationDirectoryError(OSError):
|
|
134
|
+
"""A download destination could not be safely opened, created, or
|
|
135
|
+
written to — including a fail-closed refusal of a symlink/junction race.
|
|
136
|
+
|
|
137
|
+
An ``OSError`` subclass so every existing ``except (OSError, ValueError)``
|
|
138
|
+
around a download write (``_download_and_report`` and its Model/Output
|
|
139
|
+
counterparts in ``cli.py``) already reports it as one clean CLI line;
|
|
140
|
+
no caller needs a new except clause to handle this module's refusals.
|
|
141
|
+
"""
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _retained_temp_error(
|
|
145
|
+
destination_display_path: Path, temp_name: str, exc: Exception, *, existing_final_preserved: bool = False
|
|
146
|
+
) -> DestinationDirectoryError:
|
|
147
|
+
"""Describe a possible temporary residue without trusting its current path."""
|
|
148
|
+
preservation = "Existing final entry was preserved. " if existing_final_preserved else ""
|
|
149
|
+
return DestinationDirectoryError(
|
|
150
|
+
f"{preservation}Download created private temporary entry {temp_name!r} in the originally selected "
|
|
151
|
+
f"destination directory {destination_display_path} before the failure. That directory and name may now "
|
|
152
|
+
"resolve elsewhere; locate and verify the original entry before cleanup. Do not delete a same-named "
|
|
153
|
+
"current path unless its identity is independently confirmed. "
|
|
154
|
+
f"Could not complete the download write: {exc}."
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _close_fd_quietly(fd: int) -> None:
|
|
159
|
+
try:
|
|
160
|
+
os.close(fd)
|
|
161
|
+
except OSError:
|
|
162
|
+
pass
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
#: ``True`` when this interpreter can perform openat-style,
|
|
166
|
+
#: directory-file-descriptor-relative operations for every primitive this
|
|
167
|
+
#: module needs. A platform property, checked once at import time — used by
|
|
168
|
+
#: :func:`open_destination_directory` to pick
|
|
169
|
+
#: :class:`_DirFdDestinationDirectory` (POSIX, race-free) or
|
|
170
|
+
#: :class:`_PathIdentityDestinationDirectory` (the documented Windows
|
|
171
|
+
#: fallback, race-narrowing) — see the module docstring's WINDOWS section.
|
|
172
|
+
SUPPORTS_DIR_FD = (
|
|
173
|
+
hasattr(os, "O_DIRECTORY")
|
|
174
|
+
and hasattr(os, "O_NOFOLLOW")
|
|
175
|
+
and os.open in os.supports_dir_fd
|
|
176
|
+
and os.mkdir in os.supports_dir_fd
|
|
177
|
+
# `os.replace` itself is never listed in `os.supports_dir_fd` — CPython
|
|
178
|
+
# only advertises its sibling `os.rename` there, even though (measured:
|
|
179
|
+
# both share the same underlying `posixmodule.c` rename implementation,
|
|
180
|
+
# differing only in overwrite behavior) `os.replace(src, dst,
|
|
181
|
+
# src_dir_fd=..., dst_dir_fd=...)` works everywhere `os.rename` with
|
|
182
|
+
# those same arguments does. Check the one CPython actually documents.
|
|
183
|
+
and os.rename in os.supports_dir_fd
|
|
184
|
+
and os.unlink in os.supports_dir_fd
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
#: Bounded retries for picking an unused random temp-file name directly
|
|
188
|
+
#: inside the destination directory — mirrors ``tempfile``'s own retry
|
|
189
|
+
#: count for the same collision-on-``O_EXCL`` situation.
|
|
190
|
+
_TEMP_NAME_ATTEMPTS = 64
|
|
191
|
+
|
|
192
|
+
# Linux ``renameat2(..., RENAME_NOREPLACE)`` is one syscall: either the
|
|
193
|
+
# temporary entry becomes the final entry and the temporary name disappears,
|
|
194
|
+
# or an existing final entry is left untouched. ``link`` followed by
|
|
195
|
+
# ``unlink`` cannot offer that postcondition because a process death between
|
|
196
|
+
# those calls leaves both names behind. Keep the optional libc binding at
|
|
197
|
+
# module scope so unsupported libc/platform combinations are explicitly
|
|
198
|
+
# refused rather than silently falling back to replacement.
|
|
199
|
+
_AT_FDCWD = -100
|
|
200
|
+
_RENAME_NOREPLACE = 1
|
|
201
|
+
_RENAME_EXCL = 0x00000004
|
|
202
|
+
_renameat2 = None
|
|
203
|
+
_renameatx_np = None
|
|
204
|
+
if sys.platform.startswith("linux"):
|
|
205
|
+
_libc = ctypes.CDLL(None, use_errno=True)
|
|
206
|
+
_renameat2 = getattr(_libc, "renameat2", None)
|
|
207
|
+
if _renameat2 is not None:
|
|
208
|
+
_renameat2.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
|
|
209
|
+
_renameat2.restype = ctypes.c_int
|
|
210
|
+
elif sys.platform == "darwin":
|
|
211
|
+
_libc = ctypes.CDLL(None, use_errno=True)
|
|
212
|
+
_renameatx_np = getattr(_libc, "renameatx_np", None)
|
|
213
|
+
if _renameatx_np is not None:
|
|
214
|
+
_renameatx_np.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
|
|
215
|
+
_renameatx_np.restype = ctypes.c_int
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _rename_noreplace(
|
|
219
|
+
source: str,
|
|
220
|
+
destination: str,
|
|
221
|
+
*,
|
|
222
|
+
src_dir_fd: Optional[int] = None,
|
|
223
|
+
dst_dir_fd: Optional[int] = None,
|
|
224
|
+
) -> None:
|
|
225
|
+
"""Atomically publish *source* at absent *destination*, or fail closed.
|
|
226
|
+
|
|
227
|
+
Linux uses ``renameat2(RENAME_NOREPLACE)`` and macOS uses
|
|
228
|
+
``renameatx_np(RENAME_EXCL)``. Windows' native ``os.rename`` refuses an
|
|
229
|
+
existing destination. Other platforms deliberately refuse the
|
|
230
|
+
server-derived no-clobber path instead of using a two-name link/unlink
|
|
231
|
+
sequence or replacing an existing file. Explicit user file targets keep
|
|
232
|
+
using ``os.replace`` through ``replace_existing=True``.
|
|
233
|
+
"""
|
|
234
|
+
if os.name == "nt":
|
|
235
|
+
if src_dir_fd is not None or dst_dir_fd is not None:
|
|
236
|
+
raise DestinationDirectoryError(
|
|
237
|
+
"This platform cannot atomically publish a server-named download through directory descriptors."
|
|
238
|
+
)
|
|
239
|
+
os.rename(source, destination)
|
|
240
|
+
return
|
|
241
|
+
if sys.platform.startswith("linux"):
|
|
242
|
+
rename_function = _renameat2
|
|
243
|
+
flag = _RENAME_NOREPLACE
|
|
244
|
+
elif sys.platform == "darwin":
|
|
245
|
+
rename_function = _renameatx_np
|
|
246
|
+
flag = _RENAME_EXCL
|
|
247
|
+
else:
|
|
248
|
+
rename_function = None
|
|
249
|
+
flag = 0
|
|
250
|
+
if rename_function is None:
|
|
251
|
+
raise DestinationDirectoryError(
|
|
252
|
+
"This platform cannot atomically publish a server-named download without replacing an existing file."
|
|
253
|
+
)
|
|
254
|
+
old_dir_fd = _AT_FDCWD if src_dir_fd is None else src_dir_fd
|
|
255
|
+
new_dir_fd = _AT_FDCWD if dst_dir_fd is None else dst_dir_fd
|
|
256
|
+
ctypes.set_errno(0)
|
|
257
|
+
result = rename_function(
|
|
258
|
+
old_dir_fd,
|
|
259
|
+
os.fsencode(source),
|
|
260
|
+
new_dir_fd,
|
|
261
|
+
os.fsencode(destination),
|
|
262
|
+
flag,
|
|
263
|
+
)
|
|
264
|
+
if result == 0:
|
|
265
|
+
return
|
|
266
|
+
error_number = ctypes.get_errno()
|
|
267
|
+
if error_number == errno.EEXIST:
|
|
268
|
+
raise FileExistsError(error_number, os.strerror(error_number), destination)
|
|
269
|
+
if error_number in (errno.EINVAL, errno.ENOSYS, errno.EOPNOTSUPP):
|
|
270
|
+
raise DestinationDirectoryError(
|
|
271
|
+
"This filesystem cannot atomically publish a server-named download without replacing an existing file."
|
|
272
|
+
)
|
|
273
|
+
raise OSError(error_number, os.strerror(error_number), destination)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _random_temp_name(final_name: str) -> str:
|
|
277
|
+
"""One candidate temp-file name for ``final_name``, unpredictable
|
|
278
|
+
enough that an attacker cannot pre-plant a symlink at it — the
|
|
279
|
+
surrounding directory's OWN identity is what this module protects;
|
|
280
|
+
the temp name's job is only to avoid colliding with anything already
|
|
281
|
+
there, exactly like ``tempfile.mkstemp``."""
|
|
282
|
+
return f".{final_name}.{secrets.token_hex(8)}.part"
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _portable_name_key(name: str) -> str:
|
|
286
|
+
return unicodedata.normalize("NFKC", name).casefold()
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _validate_leaf_name(final_name: str) -> None:
|
|
290
|
+
"""Refuse anything that is not a bare basename.
|
|
291
|
+
|
|
292
|
+
Every documented caller already gets a bare basename from
|
|
293
|
+
``_safe_download_filename`` before it ever reaches here, but the
|
|
294
|
+
containment this module sells ("cannot land outside the directory the
|
|
295
|
+
handle identifies, by construction") is only true if this precondition
|
|
296
|
+
actually holds — checked explicitly rather than left as an unenforced
|
|
297
|
+
assumption a future caller could violate silently.
|
|
298
|
+
"""
|
|
299
|
+
if not final_name or final_name in (".", "..") or "/" in final_name:
|
|
300
|
+
raise DestinationDirectoryError(f"Refusing to write {final_name!r}: not a safe bare filename.")
|
|
301
|
+
if os.sep != "/" and os.sep in final_name:
|
|
302
|
+
raise DestinationDirectoryError(f"Refusing to write {final_name!r}: not a safe bare filename.")
|
|
303
|
+
if os.altsep and os.altsep in final_name:
|
|
304
|
+
raise DestinationDirectoryError(f"Refusing to write {final_name!r}: not a safe bare filename.")
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _is_reparse_point(info: os.stat_result) -> bool:
|
|
308
|
+
"""``True`` for a Windows reparse point — a directory JUNCTION
|
|
309
|
+
(``mklink /J``, no elevated privilege required) among them.
|
|
310
|
+
|
|
311
|
+
``stat.S_ISLNK`` does not cover a junction (it is not the "symlink"
|
|
312
|
+
reparse type NTFS uses), so a check that only calls ``S_ISLNK`` misses
|
|
313
|
+
the one unprivileged local-filesystem redirection primitive an
|
|
314
|
+
unprivileged Windows attacker can actually create — a true Windows
|
|
315
|
+
symlink needs ``SeCreateSymbolicLinkPrivilege``, which an ordinary local
|
|
316
|
+
account does not have by default. ``st_reparse_tag`` is populated only
|
|
317
|
+
on Windows and is ``0``/absent everywhere else, so ``getattr(..., 0)``
|
|
318
|
+
is a portable, no-op check on POSIX. ``os.path.isjunction`` would be the
|
|
319
|
+
stdlib spelling of this, but it is 3.12+ and this package's floor is
|
|
320
|
+
3.11.
|
|
321
|
+
"""
|
|
322
|
+
return getattr(info, "st_reparse_tag", 0) != 0
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _refuse_degenerate_identity(identity: tuple[int, int], expanded: Path) -> None:
|
|
326
|
+
"""Refuse a ``(0, 0)`` device/inode pair rather than silently trusting
|
|
327
|
+
it as a real identity.
|
|
328
|
+
|
|
329
|
+
Some filesystems — certain Windows network shares among them — report
|
|
330
|
+
``st_dev``/``st_ino`` as ``0`` for every entry. An identity check built
|
|
331
|
+
on that value would compare ``(0, 0) == (0, 0)`` for two UNRELATED
|
|
332
|
+
directories and pass every time, making the whole re-verification an
|
|
333
|
+
inert no-op precisely where a UNC ``-o \\\\server\\share\\dl`` style
|
|
334
|
+
destination would otherwise rely on it most.
|
|
335
|
+
"""
|
|
336
|
+
if identity == (0, 0):
|
|
337
|
+
raise DestinationDirectoryError(
|
|
338
|
+
f"Refusing to use {expanded} as a download destination: the filesystem reports a "
|
|
339
|
+
"degenerate (0, 0) device/inode identity for it (seen on some network shares), which "
|
|
340
|
+
"cannot detect a swap. Choose a destination on a filesystem that reports real identities."
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
class DestinationDirectory:
|
|
345
|
+
"""A stable handle to one download destination directory.
|
|
346
|
+
|
|
347
|
+
Acquired once, immediately before the write, by
|
|
348
|
+
:func:`open_destination_directory`; the write for that download must go
|
|
349
|
+
through :meth:`write_atomic` on this SAME instance, never through a
|
|
350
|
+
fresh path-string lookup. Use as a context manager, or call
|
|
351
|
+
:meth:`close` explicitly once the write (success or failure) is
|
|
352
|
+
complete.
|
|
353
|
+
"""
|
|
354
|
+
|
|
355
|
+
#: The path this handle was opened for — display/error text only, never
|
|
356
|
+
#: re-resolved for a filesystem operation once the handle exists.
|
|
357
|
+
display_path: Path
|
|
358
|
+
|
|
359
|
+
def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
|
|
360
|
+
"""Write *data* into ``final_name`` directly inside this directory.
|
|
361
|
+
|
|
362
|
+
A same-directory temporary file is written, flushed, and ``fsync``'d,
|
|
363
|
+
then published atomically. With ``replace_existing=True`` it replaces
|
|
364
|
+
a prior complete entry; otherwise publication fails if *any* final
|
|
365
|
+
entry already exists. Any failure after temporary-file creation may
|
|
366
|
+
leave that mode-0600 ``.part`` entry behind. No backend deletes it
|
|
367
|
+
automatically because a pathname can be replaced before cleanup.
|
|
368
|
+
Ordinary write and publication errors name the temporary entry and ask
|
|
369
|
+
the operator to locate and verify it before manual cleanup.
|
|
370
|
+
``KeyboardInterrupt`` and
|
|
371
|
+
``SystemExit`` are re-raised unchanged; their possible residue is
|
|
372
|
+
documented here rather than reported while interrupting the caller.
|
|
373
|
+
``final_name`` must be a bare basename (no path separators); see
|
|
374
|
+
:func:`_validate_leaf_name`.
|
|
375
|
+
"""
|
|
376
|
+
raise NotImplementedError
|
|
377
|
+
|
|
378
|
+
def close(self) -> None:
|
|
379
|
+
raise NotImplementedError
|
|
380
|
+
|
|
381
|
+
def __enter__(self) -> "DestinationDirectory":
|
|
382
|
+
return self
|
|
383
|
+
|
|
384
|
+
def __exit__(self, *exc_info: object) -> None:
|
|
385
|
+
self.close()
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
class _DirFdDestinationDirectory(DestinationDirectory):
|
|
389
|
+
"""POSIX ``openat()``-style handle: race-free by construction.
|
|
390
|
+
|
|
391
|
+
``fd`` is an open, ``O_DIRECTORY`` file descriptor bound to the
|
|
392
|
+
destination's inode. Every operation below passes it as ``dir_fd=``
|
|
393
|
+
with a bare basename, so none of them ever re-walks ``display_path`` as
|
|
394
|
+
a string again — and, because a bare basename cannot contain a path
|
|
395
|
+
separator, none of them can ever land outside the directory ``fd``
|
|
396
|
+
identifies, by construction rather than by re-checking.
|
|
397
|
+
"""
|
|
398
|
+
|
|
399
|
+
def __init__(self, fd: int, display_path: Path) -> None:
|
|
400
|
+
self.display_path = display_path
|
|
401
|
+
self._fd = fd
|
|
402
|
+
self._closed = False
|
|
403
|
+
|
|
404
|
+
def _mkstemp(self, final_name: str) -> tuple[int, str]:
|
|
405
|
+
last_exc: Optional[OSError] = None
|
|
406
|
+
for _ in range(_TEMP_NAME_ATTEMPTS):
|
|
407
|
+
candidate = _random_temp_name(final_name)
|
|
408
|
+
try:
|
|
409
|
+
fd = os.open(candidate, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600, dir_fd=self._fd)
|
|
410
|
+
except FileExistsError as exc:
|
|
411
|
+
last_exc = exc
|
|
412
|
+
continue
|
|
413
|
+
except FileNotFoundError as exc:
|
|
414
|
+
# The held directory was removed out from under this
|
|
415
|
+
# handle (e.g. an attacker who deleted a genuinely
|
|
416
|
+
# pre-existing directory after it was validated) — the
|
|
417
|
+
# kernel refuses to add new entries to an unlinked
|
|
418
|
+
# directory even though this fd still refers to it, which
|
|
419
|
+
# is itself a correct refusal. Give it a clear message
|
|
420
|
+
# rather than surfacing the bare temp filename.
|
|
421
|
+
raise DestinationDirectoryError(
|
|
422
|
+
f"Refusing to write into {self.display_path}: the destination directory no longer "
|
|
423
|
+
"exists. It may have been removed after it was validated."
|
|
424
|
+
) from exc
|
|
425
|
+
return fd, candidate
|
|
426
|
+
raise DestinationDirectoryError(
|
|
427
|
+
f"Could not create a unique temporary file for {final_name!r} in {self.display_path} "
|
|
428
|
+
"after repeated attempts."
|
|
429
|
+
) from last_exc
|
|
430
|
+
|
|
431
|
+
def _refuse_portable_alias(self, final_name: str, *, published_identity: Optional[tuple[int, int]] = None) -> None:
|
|
432
|
+
aliases = [
|
|
433
|
+
name
|
|
434
|
+
for name in os.listdir(self._fd)
|
|
435
|
+
if name != final_name and _portable_name_key(name) == _portable_name_key(final_name)
|
|
436
|
+
]
|
|
437
|
+
if not aliases:
|
|
438
|
+
return
|
|
439
|
+
if published_identity is not None:
|
|
440
|
+
current = os.stat(final_name, dir_fd=self._fd, follow_symlinks=False)
|
|
441
|
+
if (current.st_dev, current.st_ino) != published_identity:
|
|
442
|
+
raise DestinationDirectoryError(
|
|
443
|
+
f"A portable-equivalent alias appeared after the complete download was published at "
|
|
444
|
+
f"{final_name!r}, and another process replaced that entry before identity verification. "
|
|
445
|
+
"Manual resolution is required; the client did not delete either entry."
|
|
446
|
+
)
|
|
447
|
+
# Python exposes no conditional unlink-by-inode primitive. Leave
|
|
448
|
+
# the proven self-published entry in place rather than reopening a
|
|
449
|
+
# stat-to-unlink race that could delete a replacement.
|
|
450
|
+
raise DestinationDirectoryError(
|
|
451
|
+
f"The complete download was published at {final_name!r}, but portable-equivalent alias "
|
|
452
|
+
f"{aliases[0]!r} appeared. Manual resolution is required; the client did not delete either entry."
|
|
453
|
+
)
|
|
454
|
+
raise DestinationDirectoryError(
|
|
455
|
+
f"Refusing to publish {final_name!r}: portable-equivalent alias {aliases[0]!r} exists."
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
|
|
459
|
+
_validate_leaf_name(final_name)
|
|
460
|
+
if not replace_existing:
|
|
461
|
+
self._refuse_portable_alias(final_name)
|
|
462
|
+
fd, temp_name = self._mkstemp(final_name)
|
|
463
|
+
try:
|
|
464
|
+
published_identity = os.fstat(fd)
|
|
465
|
+
except (KeyboardInterrupt, SystemExit):
|
|
466
|
+
_close_fd_quietly(fd)
|
|
467
|
+
raise
|
|
468
|
+
except Exception as exc:
|
|
469
|
+
_close_fd_quietly(fd)
|
|
470
|
+
raise _retained_temp_error(self.display_path, temp_name, exc) from exc
|
|
471
|
+
try:
|
|
472
|
+
handle = os.fdopen(fd, "wb")
|
|
473
|
+
except (KeyboardInterrupt, SystemExit):
|
|
474
|
+
_close_fd_quietly(fd)
|
|
475
|
+
raise
|
|
476
|
+
except Exception as exc:
|
|
477
|
+
_close_fd_quietly(fd)
|
|
478
|
+
raise _retained_temp_error(self.display_path, temp_name, exc) from exc
|
|
479
|
+
published = False
|
|
480
|
+
existing_final_preserved = False
|
|
481
|
+
try:
|
|
482
|
+
with handle:
|
|
483
|
+
handle.write(data)
|
|
484
|
+
handle.flush()
|
|
485
|
+
os.fsync(handle.fileno())
|
|
486
|
+
if replace_existing:
|
|
487
|
+
os.replace(temp_name, final_name, src_dir_fd=self._fd, dst_dir_fd=self._fd)
|
|
488
|
+
published = True
|
|
489
|
+
else:
|
|
490
|
+
try:
|
|
491
|
+
_rename_noreplace(temp_name, final_name, src_dir_fd=self._fd, dst_dir_fd=self._fd)
|
|
492
|
+
except FileExistsError as exc:
|
|
493
|
+
existing_final_preserved = True
|
|
494
|
+
raise DestinationDirectoryError(
|
|
495
|
+
f"Refusing to replace existing download destination {final_name!r} in {self.display_path}. "
|
|
496
|
+
"The existing final entry was preserved."
|
|
497
|
+
) from exc
|
|
498
|
+
published = True
|
|
499
|
+
self._refuse_portable_alias(
|
|
500
|
+
final_name, published_identity=(published_identity.st_dev, published_identity.st_ino)
|
|
501
|
+
)
|
|
502
|
+
except (KeyboardInterrupt, SystemExit):
|
|
503
|
+
raise
|
|
504
|
+
except Exception as exc:
|
|
505
|
+
if published:
|
|
506
|
+
raise
|
|
507
|
+
raise _retained_temp_error(
|
|
508
|
+
self.display_path, temp_name, exc, existing_final_preserved=existing_final_preserved
|
|
509
|
+
) from exc
|
|
510
|
+
|
|
511
|
+
def close(self) -> None:
|
|
512
|
+
if not self._closed:
|
|
513
|
+
self._closed = True
|
|
514
|
+
os.close(self._fd)
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
class _PathIdentityDestinationDirectory(DestinationDirectory):
|
|
518
|
+
"""Windows (no ``dir_fd``) fallback: narrows, but cannot eliminate, the
|
|
519
|
+
ancestor-component race. See the module docstring's WINDOWS section for
|
|
520
|
+
exactly what this class does and does not guarantee."""
|
|
521
|
+
|
|
522
|
+
def __init__(self, display_path: Path, identity: tuple[int, int], *, originally_symlink: bool) -> None:
|
|
523
|
+
self.display_path = display_path
|
|
524
|
+
#: ``(st_dev, st_ino)`` captured when this handle was opened —
|
|
525
|
+
#: re-verified before every filesystem operation below.
|
|
526
|
+
self._identity = identity
|
|
527
|
+
#: Whether *display_path* was ALREADY a symlink (to a real
|
|
528
|
+
#: directory) the first time this handle looked — an established
|
|
529
|
+
#: policy allows that, so every later check must keep following it
|
|
530
|
+
#: (and compare the TARGET's identity) rather than refusing outright
|
|
531
|
+
#: merely because the path is, as it always was, a symlink. A path
|
|
532
|
+
#: that started as a plain directory must never become one.
|
|
533
|
+
self._originally_symlink = originally_symlink
|
|
534
|
+
self._closed = False
|
|
535
|
+
|
|
536
|
+
def _verify_unchanged(self) -> None:
|
|
537
|
+
path_str = os.fspath(self.display_path)
|
|
538
|
+
follow = self._originally_symlink
|
|
539
|
+
try:
|
|
540
|
+
info = os.stat(path_str, follow_symlinks=follow)
|
|
541
|
+
except OSError as exc:
|
|
542
|
+
raise DestinationDirectoryError(
|
|
543
|
+
f"Download destination {self.display_path} became unreachable during the download: {exc}"
|
|
544
|
+
) from exc
|
|
545
|
+
if not follow and (stat_module.S_ISLNK(info.st_mode) or _is_reparse_point(info)):
|
|
546
|
+
raise DestinationDirectoryError(
|
|
547
|
+
f"Refusing to write into {self.display_path}: it became a symlink or junction during " "the download."
|
|
548
|
+
)
|
|
549
|
+
if not stat_module.S_ISDIR(info.st_mode):
|
|
550
|
+
raise DestinationDirectoryError(f"Refusing to write into {self.display_path}: it is no longer a directory.")
|
|
551
|
+
if (info.st_dev, info.st_ino) != self._identity:
|
|
552
|
+
raise DestinationDirectoryError(
|
|
553
|
+
f"Refusing to write into {self.display_path}: it was replaced with a different "
|
|
554
|
+
"directory during the download."
|
|
555
|
+
)
|
|
556
|
+
|
|
557
|
+
def _mkstemp(self, final_name: str) -> tuple[int, Path]:
|
|
558
|
+
"""Bounded retry on an ``O_EXCL`` temp-name collision — mirrors
|
|
559
|
+
:meth:`_DirFdDestinationDirectory._mkstemp`'s 64-attempt loop (and
|
|
560
|
+
the pre-fix ``tempfile.mkstemp``, which retried on every platform).
|
|
561
|
+
Without this, an uncaught ``FileExistsError`` on the ~1/2**64-odds
|
|
562
|
+
collision would simply fail the download outright instead of
|
|
563
|
+
retrying, on the one platform where this path is load-bearing.
|
|
564
|
+
"""
|
|
565
|
+
last_exc: Optional[OSError] = None
|
|
566
|
+
for _ in range(_TEMP_NAME_ATTEMPTS):
|
|
567
|
+
temp_path = self.display_path / _random_temp_name(final_name)
|
|
568
|
+
try:
|
|
569
|
+
fd = os.open(os.fspath(temp_path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
570
|
+
except FileExistsError as exc:
|
|
571
|
+
last_exc = exc
|
|
572
|
+
continue
|
|
573
|
+
return fd, temp_path
|
|
574
|
+
raise DestinationDirectoryError(
|
|
575
|
+
f"Could not create a unique temporary file for {final_name!r} in {self.display_path} "
|
|
576
|
+
"after repeated attempts."
|
|
577
|
+
) from last_exc
|
|
578
|
+
|
|
579
|
+
def _refuse_portable_alias(self, final_name: str, *, published_identity: Optional[tuple[int, int]] = None) -> None:
|
|
580
|
+
aliases = [
|
|
581
|
+
path
|
|
582
|
+
for path in self.display_path.iterdir()
|
|
583
|
+
if path.name != final_name and _portable_name_key(path.name) == _portable_name_key(final_name)
|
|
584
|
+
]
|
|
585
|
+
if not aliases:
|
|
586
|
+
return
|
|
587
|
+
if published_identity is not None:
|
|
588
|
+
final_path = self.display_path / final_name
|
|
589
|
+
info = final_path.stat(follow_symlinks=False)
|
|
590
|
+
if (info.st_dev, info.st_ino) != published_identity:
|
|
591
|
+
raise DestinationDirectoryError(
|
|
592
|
+
f"A portable-equivalent alias appeared after the complete download was published at "
|
|
593
|
+
f"{final_name!r}, and another process replaced that entry before identity verification. "
|
|
594
|
+
"Manual resolution is required; the client did not delete either entry."
|
|
595
|
+
)
|
|
596
|
+
raise DestinationDirectoryError(
|
|
597
|
+
f"The complete download was published at {final_name!r}, but portable-equivalent alias "
|
|
598
|
+
f"{aliases[0].name!r} appeared. Manual resolution is required; the client did not delete either entry."
|
|
599
|
+
)
|
|
600
|
+
raise DestinationDirectoryError(f"Refusing to publish {final_name!r}: portable-equivalent alias exists.")
|
|
601
|
+
|
|
602
|
+
def write_atomic(self, final_name: str, data: bytes, *, replace_existing: bool = True) -> None:
|
|
603
|
+
_validate_leaf_name(final_name)
|
|
604
|
+
self._verify_unchanged()
|
|
605
|
+
if not replace_existing:
|
|
606
|
+
self._refuse_portable_alias(final_name)
|
|
607
|
+
fd, temp_path = self._mkstemp(final_name)
|
|
608
|
+
try:
|
|
609
|
+
published_identity = os.fstat(fd)
|
|
610
|
+
except (KeyboardInterrupt, SystemExit):
|
|
611
|
+
_close_fd_quietly(fd)
|
|
612
|
+
raise
|
|
613
|
+
except Exception as exc:
|
|
614
|
+
_close_fd_quietly(fd)
|
|
615
|
+
raise _retained_temp_error(self.display_path, temp_path.name, exc) from exc
|
|
616
|
+
final_path = self.display_path / final_name
|
|
617
|
+
try:
|
|
618
|
+
handle = os.fdopen(fd, "wb")
|
|
619
|
+
except (KeyboardInterrupt, SystemExit):
|
|
620
|
+
_close_fd_quietly(fd)
|
|
621
|
+
raise
|
|
622
|
+
except Exception as exc:
|
|
623
|
+
_close_fd_quietly(fd)
|
|
624
|
+
raise _retained_temp_error(self.display_path, temp_path.name, exc) from exc
|
|
625
|
+
published = False
|
|
626
|
+
existing_final_preserved = False
|
|
627
|
+
try:
|
|
628
|
+
with handle:
|
|
629
|
+
handle.write(data)
|
|
630
|
+
handle.flush()
|
|
631
|
+
os.fsync(handle.fileno())
|
|
632
|
+
self._verify_unchanged()
|
|
633
|
+
if replace_existing:
|
|
634
|
+
os.replace(os.fspath(temp_path), os.fspath(final_path))
|
|
635
|
+
published = True
|
|
636
|
+
else:
|
|
637
|
+
try:
|
|
638
|
+
_rename_noreplace(os.fspath(temp_path), os.fspath(final_path))
|
|
639
|
+
except FileExistsError as exc:
|
|
640
|
+
existing_final_preserved = True
|
|
641
|
+
raise DestinationDirectoryError(
|
|
642
|
+
f"Refusing to replace existing download destination {final_name!r} in {self.display_path}. "
|
|
643
|
+
"The existing final entry was preserved."
|
|
644
|
+
) from exc
|
|
645
|
+
published = True
|
|
646
|
+
self._refuse_portable_alias(
|
|
647
|
+
final_name, published_identity=(published_identity.st_dev, published_identity.st_ino)
|
|
648
|
+
)
|
|
649
|
+
except (KeyboardInterrupt, SystemExit):
|
|
650
|
+
raise
|
|
651
|
+
except Exception as exc:
|
|
652
|
+
if published:
|
|
653
|
+
raise
|
|
654
|
+
raise _retained_temp_error(
|
|
655
|
+
self.display_path, temp_path.name, exc, existing_final_preserved=existing_final_preserved
|
|
656
|
+
) from exc
|
|
657
|
+
|
|
658
|
+
def close(self) -> None:
|
|
659
|
+
self._closed = True
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
# ---------------------------------------------------------------------------
|
|
663
|
+
# Phase 1: observe (read-only, before any network I/O)
|
|
664
|
+
# ---------------------------------------------------------------------------
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
class _ComponentObservation:
|
|
668
|
+
"""One path component's state as observed, before the fetch: its name,
|
|
669
|
+
and either the resolved (device, inode) identity it had (existed), or
|
|
670
|
+
``None`` (did not exist — and, transitively, neither did anything below
|
|
671
|
+
it)."""
|
|
672
|
+
|
|
673
|
+
__slots__ = ("name", "identity")
|
|
674
|
+
|
|
675
|
+
def __init__(self, name: str, identity: Optional[tuple[int, int]]) -> None:
|
|
676
|
+
self.name = name
|
|
677
|
+
self.identity = identity
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
class DestinationObservation:
|
|
681
|
+
"""Opaque snapshot of a destination directory's pre-fetch state.
|
|
682
|
+
|
|
683
|
+
Produced by :func:`observe_destination`, consumed by
|
|
684
|
+
:func:`open_destination_directory`. Never constructed directly, and
|
|
685
|
+
never partially reused across two different destinations — call
|
|
686
|
+
:func:`observe_destination` again for a different path.
|
|
687
|
+
|
|
688
|
+
On POSIX, this may hold an OPEN file descriptor for the deepest
|
|
689
|
+
existing path component (``held_fd``) for the entire gap between
|
|
690
|
+
observation and open — see the module docstring for why a held
|
|
691
|
+
descriptor, not a recorded identity value, is what closes the race for
|
|
692
|
+
that prefix. Callers MUST call :meth:`close` (or use this as a context
|
|
693
|
+
manager) on every path, including a digest mismatch or a failed fetch,
|
|
694
|
+
so that descriptor is never leaked. :func:`open_destination_directory`
|
|
695
|
+
takes ownership of it on success — a redundant :meth:`close` afterward
|
|
696
|
+
is always safe.
|
|
697
|
+
"""
|
|
698
|
+
|
|
699
|
+
def __init__(
|
|
700
|
+
self,
|
|
701
|
+
expanded: Path,
|
|
702
|
+
*,
|
|
703
|
+
is_absolute: bool,
|
|
704
|
+
held_fd: Optional[int] = None,
|
|
705
|
+
absent_names: Optional[list[str]] = None,
|
|
706
|
+
components: Optional[list[_ComponentObservation]] = None,
|
|
707
|
+
existed: bool = True,
|
|
708
|
+
identity: Optional[tuple[int, int]] = None,
|
|
709
|
+
was_symlink: bool = False,
|
|
710
|
+
) -> None:
|
|
711
|
+
self.expanded = expanded
|
|
712
|
+
self.is_absolute = is_absolute
|
|
713
|
+
#: POSIX only: an open fd for the deepest EXISTING component, or
|
|
714
|
+
#: ``None`` if nothing below the anchor existed (a relative
|
|
715
|
+
#: destination with no existing prefix at all — the anchor is then
|
|
716
|
+
#: the process's own current working directory, itself immune to a
|
|
717
|
+
#: path-string race; see :func:`_open_posix`). Ownership transfers
|
|
718
|
+
#: to the :class:`DestinationDirectory` :func:`open_destination_directory`
|
|
719
|
+
#: returns on success.
|
|
720
|
+
self.held_fd = held_fd
|
|
721
|
+
#: POSIX only: the ordered suffix of component names that did NOT
|
|
722
|
+
#: exist at observation time and must be created (each guarded by
|
|
723
|
+
#: ``O_NOFOLLOW`` at open time).
|
|
724
|
+
self.absent_names = absent_names
|
|
725
|
+
#: The full per-component chain, Windows-fallback only (``None`` on
|
|
726
|
+
#: POSIX, which uses ``held_fd``/``absent_names`` instead — see the
|
|
727
|
+
#: module docstring's WINDOWS section for why the fallback cannot
|
|
728
|
+
#: use a held handle the same way).
|
|
729
|
+
self.components = components
|
|
730
|
+
#: Convenience mirror of the FINAL component's own state — fallback only.
|
|
731
|
+
self.existed = existed
|
|
732
|
+
self.identity = identity
|
|
733
|
+
self.was_symlink = was_symlink
|
|
734
|
+
self._closed = False
|
|
735
|
+
self._consumed = False
|
|
736
|
+
|
|
737
|
+
def _take_for_open(self) -> None:
|
|
738
|
+
"""Claim this observation for exactly one
|
|
739
|
+
:func:`open_destination_directory` call, refusing reuse.
|
|
740
|
+
|
|
741
|
+
A containment primitive whose misuse mode is "silently create and
|
|
742
|
+
write in a DIFFERENT directory" is the wrong direction to fail.
|
|
743
|
+
Without this guard, a second open on an already-spent observation
|
|
744
|
+
(``held_fd`` already handed off, ``None``) falls through to an
|
|
745
|
+
unqualified, ``dir_fd=None`` — i.e. CWD-relative — walk on POSIX,
|
|
746
|
+
landing bytes wherever the process happens to be running rather
|
|
747
|
+
than the validated destination; opening after :meth:`close` raises
|
|
748
|
+
a raw, non-``OSError`` exception (an ``AssertionError``, or under
|
|
749
|
+
``python -O`` a ``TypeError`` from operating on a ``None`` fd) that
|
|
750
|
+
the CLI's ``except (OSError, ValueError)`` does not catch. Refusing
|
|
751
|
+
outright, the same way for both misuse shapes, is strictly safer
|
|
752
|
+
than either.
|
|
753
|
+
"""
|
|
754
|
+
if self._consumed:
|
|
755
|
+
raise DestinationDirectoryError(
|
|
756
|
+
f"Refusing to open {self.expanded}: this observation was already used for a previous "
|
|
757
|
+
"open_destination_directory() call, or was closed first. Call observe_destination() "
|
|
758
|
+
"again to snapshot the destination and attempt a new open."
|
|
759
|
+
)
|
|
760
|
+
self._consumed = True
|
|
761
|
+
|
|
762
|
+
def is_existing_directory(self) -> bool:
|
|
763
|
+
"""Whether the selected directory existed when this observation began."""
|
|
764
|
+
if self.absent_names is not None:
|
|
765
|
+
return not self.absent_names
|
|
766
|
+
return self.existed
|
|
767
|
+
|
|
768
|
+
def matches_directory(self, directory: Path) -> bool:
|
|
769
|
+
"""Whether *directory* is exactly the path this observation owns."""
|
|
770
|
+
return self.expanded == Path(os.path.expanduser(os.fspath(directory)))
|
|
771
|
+
|
|
772
|
+
def contained_destination(self, final_name: str) -> Path:
|
|
773
|
+
"""Return a final path that passed this observation's containment check.
|
|
774
|
+
|
|
775
|
+
The later open consumes this same observation, so a selected directory
|
|
776
|
+
swapped after this check cannot redirect the write. A swap visible to
|
|
777
|
+
the path-resolution check instead causes a refusal.
|
|
778
|
+
"""
|
|
779
|
+
_validate_leaf_name(final_name)
|
|
780
|
+
destination = self.expanded / final_name
|
|
781
|
+
try:
|
|
782
|
+
resolved_root = self.expanded.resolve(strict=False)
|
|
783
|
+
resolved_destination = destination.resolve(strict=False)
|
|
784
|
+
resolved_destination.relative_to(resolved_root)
|
|
785
|
+
except (OSError, RuntimeError, ValueError) as exc:
|
|
786
|
+
raise DestinationDirectoryError(
|
|
787
|
+
f"Could not validate {destination} beneath selected directory {self.expanded}: {exc}"
|
|
788
|
+
) from exc
|
|
789
|
+
if resolved_destination == resolved_root:
|
|
790
|
+
raise DestinationDirectoryError(
|
|
791
|
+
f"Refusing to write {final_name!r}: it resolves to the selected directory itself."
|
|
792
|
+
)
|
|
793
|
+
return destination
|
|
794
|
+
|
|
795
|
+
def close(self) -> None:
|
|
796
|
+
"""Release the held descriptor, if any, and mark this observation
|
|
797
|
+
spent — a later :func:`open_destination_directory` call on it must
|
|
798
|
+
refuse outright (see :meth:`_take_for_open`) rather than silently
|
|
799
|
+
falling through to an unqualified, CWD-relative walk. Idempotent
|
|
800
|
+
and safe to call after ownership has already transferred to a
|
|
801
|
+
:class:`DestinationDirectory` (``held_fd`` is ``None`` by then)."""
|
|
802
|
+
if not self._closed:
|
|
803
|
+
self._closed = True
|
|
804
|
+
self._consumed = True
|
|
805
|
+
if self.held_fd is not None:
|
|
806
|
+
os.close(self.held_fd)
|
|
807
|
+
self.held_fd = None
|
|
808
|
+
|
|
809
|
+
def __enter__(self) -> "DestinationObservation":
|
|
810
|
+
return self
|
|
811
|
+
|
|
812
|
+
def __exit__(self, *exc_info: object) -> None:
|
|
813
|
+
self.close()
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def _observe_posix(directory: Path) -> DestinationObservation:
|
|
817
|
+
expanded = Path(os.path.expanduser(os.fspath(directory)))
|
|
818
|
+
is_absolute = expanded.is_absolute()
|
|
819
|
+
parts = expanded.parts
|
|
820
|
+
remaining = parts[1:] if is_absolute else parts
|
|
821
|
+
absent_names: list[str] = []
|
|
822
|
+
#: The deepest component confirmed to exist so far — ``None`` means
|
|
823
|
+
#: "relative to the process's own current working directory" (either
|
|
824
|
+
#: because *expanded* is relative and nothing in it exists yet, or
|
|
825
|
+
#: because the walk has not opened anything below the anchor at all).
|
|
826
|
+
current_fd: Optional[int] = None
|
|
827
|
+
stop = False
|
|
828
|
+
try:
|
|
829
|
+
if is_absolute:
|
|
830
|
+
try:
|
|
831
|
+
current_fd = os.open(parts[0], os.O_DIRECTORY | os.O_RDONLY)
|
|
832
|
+
except OSError as exc:
|
|
833
|
+
raise DestinationDirectoryError(f"Could not observe download destination {expanded}: {exc}") from exc
|
|
834
|
+
for name in remaining:
|
|
835
|
+
if stop:
|
|
836
|
+
absent_names.append(name)
|
|
837
|
+
continue
|
|
838
|
+
try:
|
|
839
|
+
next_fd = os.open(name, os.O_DIRECTORY | os.O_RDONLY, dir_fd=current_fd)
|
|
840
|
+
except FileNotFoundError:
|
|
841
|
+
absent_names.append(name)
|
|
842
|
+
stop = True
|
|
843
|
+
continue
|
|
844
|
+
except NotADirectoryError as exc:
|
|
845
|
+
raise DestinationDirectoryError(f"{expanded}: {name!r} exists and is not a directory.") from exc
|
|
846
|
+
except OSError as exc:
|
|
847
|
+
raise DestinationDirectoryError(f"Could not observe download destination {expanded}: {exc}") from exc
|
|
848
|
+
# `next_fd` becomes the new deepest-known-existing component —
|
|
849
|
+
# release the fd it superseded (never the one we are about to
|
|
850
|
+
# return) BEFORE advancing, so a later exception in this loop
|
|
851
|
+
# closes at most one live fd via the `except` below, never two.
|
|
852
|
+
if current_fd is not None:
|
|
853
|
+
os.close(current_fd)
|
|
854
|
+
current_fd = next_fd
|
|
855
|
+
except BaseException:
|
|
856
|
+
if current_fd is not None:
|
|
857
|
+
os.close(current_fd)
|
|
858
|
+
raise
|
|
859
|
+
# `current_fd` is now the deepest EXISTING component's fd (or the
|
|
860
|
+
# anchor's, or None for "nothing exists, relative to cwd") — HELD open,
|
|
861
|
+
# deliberately, not closed: this is what pins its inode against reuse
|
|
862
|
+
# for the entire gap until `open_destination_directory` runs. See the
|
|
863
|
+
# module docstring.
|
|
864
|
+
was_symlink = False
|
|
865
|
+
if not absent_names:
|
|
866
|
+
try:
|
|
867
|
+
was_symlink = stat_module.S_ISLNK(os.lstat(os.fspath(expanded)).st_mode)
|
|
868
|
+
except OSError as exc:
|
|
869
|
+
if current_fd is not None:
|
|
870
|
+
os.close(current_fd)
|
|
871
|
+
raise DestinationDirectoryError(f"Could not observe download destination {expanded}: {exc}") from exc
|
|
872
|
+
return DestinationObservation(
|
|
873
|
+
expanded,
|
|
874
|
+
is_absolute=is_absolute,
|
|
875
|
+
held_fd=current_fd,
|
|
876
|
+
absent_names=absent_names,
|
|
877
|
+
was_symlink=was_symlink,
|
|
878
|
+
)
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def _observe_fallback(directory: Path) -> DestinationObservation:
|
|
882
|
+
"""The Windows-fallback observation: no ``dir_fd`` to hold or chain, so
|
|
883
|
+
each component is checked by ``os.stat``ing the ACCUMULATED path
|
|
884
|
+
string built up so far, and its (device, inode) VALUE is recorded —
|
|
885
|
+
weaker than the POSIX walk (which holds the deepest existing
|
|
886
|
+
component's descriptor open instead of trusting a recorded value; see
|
|
887
|
+
the module docstring for why a value can be fooled by inode reuse) but
|
|
888
|
+
still a real per-component snapshot, closing the same ancestor gap the
|
|
889
|
+
POSIX walk closes, just not to zero width — see the module docstring's
|
|
890
|
+
WINDOWS section for exactly what residual window this leaves.
|
|
891
|
+
"""
|
|
892
|
+
expanded = Path(os.path.expanduser(os.fspath(directory)))
|
|
893
|
+
is_absolute = expanded.is_absolute()
|
|
894
|
+
parts = expanded.parts
|
|
895
|
+
remaining = parts[1:] if is_absolute else parts
|
|
896
|
+
components: list[_ComponentObservation] = []
|
|
897
|
+
accumulated = Path(parts[0]) if is_absolute else Path(".")
|
|
898
|
+
stop = False
|
|
899
|
+
for name in remaining:
|
|
900
|
+
accumulated = accumulated / name
|
|
901
|
+
if stop:
|
|
902
|
+
components.append(_ComponentObservation(name, None))
|
|
903
|
+
continue
|
|
904
|
+
try:
|
|
905
|
+
info = os.stat(os.fspath(accumulated), follow_symlinks=True)
|
|
906
|
+
except FileNotFoundError:
|
|
907
|
+
components.append(_ComponentObservation(name, None))
|
|
908
|
+
stop = True
|
|
909
|
+
continue
|
|
910
|
+
except OSError as exc:
|
|
911
|
+
raise DestinationDirectoryError(f"Could not observe download destination {expanded}: {exc}") from exc
|
|
912
|
+
if not stat_module.S_ISDIR(info.st_mode):
|
|
913
|
+
raise DestinationDirectoryError(f"{expanded}: {name!r} exists and is not a directory.")
|
|
914
|
+
identity = (info.st_dev, info.st_ino)
|
|
915
|
+
_refuse_degenerate_identity(identity, expanded)
|
|
916
|
+
components.append(_ComponentObservation(name, identity))
|
|
917
|
+
|
|
918
|
+
leaf = components[-1] if components else None
|
|
919
|
+
existed = leaf.identity is not None if leaf is not None else True
|
|
920
|
+
leaf_identity = leaf.identity if leaf is not None else None
|
|
921
|
+
was_symlink = False
|
|
922
|
+
if existed:
|
|
923
|
+
try:
|
|
924
|
+
leaf_lstat = os.lstat(os.fspath(expanded))
|
|
925
|
+
except OSError as exc:
|
|
926
|
+
raise DestinationDirectoryError(f"Could not observe download destination {expanded}: {exc}") from exc
|
|
927
|
+
if _is_reparse_point(leaf_lstat):
|
|
928
|
+
raise DestinationDirectoryError(
|
|
929
|
+
f"Refusing to use {expanded} as a download destination: it is a directory junction, "
|
|
930
|
+
"which this module does not treat as a followable selected-directory symlink."
|
|
931
|
+
)
|
|
932
|
+
was_symlink = stat_module.S_ISLNK(leaf_lstat.st_mode)
|
|
933
|
+
return DestinationObservation(
|
|
934
|
+
expanded,
|
|
935
|
+
is_absolute=is_absolute,
|
|
936
|
+
components=components,
|
|
937
|
+
existed=existed,
|
|
938
|
+
identity=leaf_identity,
|
|
939
|
+
was_symlink=was_symlink,
|
|
940
|
+
)
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
def observe_destination(directory: Path) -> DestinationObservation:
|
|
944
|
+
"""Snapshot *directory*'s current state for a download about to begin.
|
|
945
|
+
|
|
946
|
+
Call this FIRST — before any network I/O for the download, including
|
|
947
|
+
resolving a presigned URL — and hold the result until bytes are ready
|
|
948
|
+
to write, then pass it to :func:`open_destination_directory`. This call
|
|
949
|
+
is read-only: it CREATES nothing, so it adds no new filesystem footprint
|
|
950
|
+
to a fetch that ultimately fails (a digest mismatch, a network error) —
|
|
951
|
+
that guarantee is unchanged from before this module existed. On POSIX
|
|
952
|
+
it may hold an open file descriptor for the deepest existing path
|
|
953
|
+
component until the returned :class:`DestinationObservation` is closed
|
|
954
|
+
(see its docstring) — a process-local resource, not a footprint on
|
|
955
|
+
disk, but the caller MUST close it on every path, success or failure.
|
|
956
|
+
|
|
957
|
+
Relative paths and ``~`` expansion are both honored, matching every
|
|
958
|
+
existing ``-o PATH``/implicit-cwd download destination.
|
|
959
|
+
"""
|
|
960
|
+
if SUPPORTS_DIR_FD:
|
|
961
|
+
return _observe_posix(directory)
|
|
962
|
+
return _observe_fallback(directory)
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
# ---------------------------------------------------------------------------
|
|
966
|
+
# Phase 2: open (immediately before the write, using the observation)
|
|
967
|
+
# ---------------------------------------------------------------------------
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def _create_absent_component(parent_fd: Optional[int], name: str, expanded: Path) -> int:
|
|
971
|
+
"""Create ``name`` directly inside ``parent_fd`` (``None`` means
|
|
972
|
+
"relative to the process's own current working directory", itself
|
|
973
|
+
immune to a path-string race), never following anything already there.
|
|
974
|
+
|
|
975
|
+
A bare ``os.mkdir`` is attempted first (fails outright — symlink or
|
|
976
|
+
not — if anything already occupies the name); whatever is then opened
|
|
977
|
+
is opened ``O_NOFOLLOW``. Since the observation already proved nothing
|
|
978
|
+
was here when the operation began, anything other than a plain
|
|
979
|
+
directory now is provably a race and is refused rather than followed.
|
|
980
|
+
|
|
981
|
+
There is no "observed PRESENT" counterpart to this function on the
|
|
982
|
+
``dir_fd`` path: a component that existed at observation time is never
|
|
983
|
+
re-walked here at all — :func:`_open_posix` continues directly from the
|
|
984
|
+
file descriptor :func:`observe_destination` kept open for it. See the
|
|
985
|
+
module docstring for why a held descriptor, not a re-checked identity
|
|
986
|
+
value, is what closes the race for that prefix.
|
|
987
|
+
"""
|
|
988
|
+
try:
|
|
989
|
+
os.mkdir(name, dir_fd=parent_fd)
|
|
990
|
+
except FileExistsError:
|
|
991
|
+
pass # benign concurrent create OR an attacker's plant -- the NOFOLLOW open below decides
|
|
992
|
+
except OSError as exc:
|
|
993
|
+
raise DestinationDirectoryError(f"Could not create download destination {expanded}: {exc}") from exc
|
|
994
|
+
try:
|
|
995
|
+
return os.open(name, os.O_DIRECTORY | os.O_RDONLY | os.O_NOFOLLOW, dir_fd=parent_fd)
|
|
996
|
+
except OSError as exc:
|
|
997
|
+
raise DestinationDirectoryError(
|
|
998
|
+
f"Refusing to use {expanded} as a download destination: {name!r} was replaced with a "
|
|
999
|
+
"symlink (or another non-directory) after it was validated as missing. This can happen "
|
|
1000
|
+
"when another process races the same path — pick a different -o PATH, or remove the "
|
|
1001
|
+
"conflicting entry and retry."
|
|
1002
|
+
) from exc
|
|
1003
|
+
|
|
1004
|
+
|
|
1005
|
+
def _open_posix(observation: DestinationObservation) -> _DirFdDestinationDirectory:
|
|
1006
|
+
expanded = observation.expanded
|
|
1007
|
+
assert observation.absent_names is not None # POSIX observations always populate this
|
|
1008
|
+
|
|
1009
|
+
# Take ownership of the held descriptor immediately: from this point,
|
|
1010
|
+
# `observation.close()` must not touch it (it is about to become either
|
|
1011
|
+
# this walk's own `current_fd`, closed on failure right here, or the
|
|
1012
|
+
# returned handle's fd) — a caller that calls `observation.close()`
|
|
1013
|
+
# again afterward, in an outer `finally`, must see a safe no-op.
|
|
1014
|
+
current_fd = observation.held_fd
|
|
1015
|
+
observation.held_fd = None
|
|
1016
|
+
try:
|
|
1017
|
+
if current_fd is not None and not observation.absent_names:
|
|
1018
|
+
try:
|
|
1019
|
+
leaf_lstat = os.lstat(os.fspath(expanded))
|
|
1020
|
+
leaf_stat = os.stat(os.fspath(expanded), follow_symlinks=True)
|
|
1021
|
+
except OSError as exc:
|
|
1022
|
+
raise DestinationDirectoryError(
|
|
1023
|
+
f"Download destination {expanded} became unreachable during the download: {exc}"
|
|
1024
|
+
) from exc
|
|
1025
|
+
if not observation.was_symlink and stat_module.S_ISLNK(leaf_lstat.st_mode):
|
|
1026
|
+
raise DestinationDirectoryError(
|
|
1027
|
+
f"Refusing to write into {expanded}: it became a symlink during the download."
|
|
1028
|
+
)
|
|
1029
|
+
held_stat = os.fstat(current_fd)
|
|
1030
|
+
if (leaf_stat.st_dev, leaf_stat.st_ino) != (held_stat.st_dev, held_stat.st_ino):
|
|
1031
|
+
raise DestinationDirectoryError(
|
|
1032
|
+
f"Refusing to write into {expanded}: it was replaced with a different directory during the download."
|
|
1033
|
+
)
|
|
1034
|
+
if current_fd is None and not observation.is_absolute:
|
|
1035
|
+
# Nothing existed below the anchor, and the anchor itself is
|
|
1036
|
+
# the process's own current working directory (an absolute
|
|
1037
|
+
# destination always has at least the root's own fd held —
|
|
1038
|
+
# see `_observe_posix`). `dir_fd=None` below already means
|
|
1039
|
+
# "resolve relative to cwd" to the kernel, which is bound to
|
|
1040
|
+
# this process's `chdir()` state, not a re-walked path string
|
|
1041
|
+
# — so this needs no fd of its own to stay race-free; opening
|
|
1042
|
+
# one explicitly is just for a uniform, always-holds-an-fd
|
|
1043
|
+
# `_DirFdDestinationDirectory` instead of a special `None` case
|
|
1044
|
+
# threaded through every one of its methods.
|
|
1045
|
+
current_fd = os.open(".", os.O_DIRECTORY | os.O_RDONLY)
|
|
1046
|
+
for name in observation.absent_names:
|
|
1047
|
+
next_fd = _create_absent_component(current_fd, name, expanded)
|
|
1048
|
+
if current_fd is not None:
|
|
1049
|
+
os.close(current_fd)
|
|
1050
|
+
current_fd = next_fd
|
|
1051
|
+
except BaseException:
|
|
1052
|
+
if current_fd is not None:
|
|
1053
|
+
os.close(current_fd)
|
|
1054
|
+
raise
|
|
1055
|
+
assert current_fd is not None
|
|
1056
|
+
return _DirFdDestinationDirectory(current_fd, expanded)
|
|
1057
|
+
|
|
1058
|
+
|
|
1059
|
+
def _open_fallback_component(accumulated: Path, comp: _ComponentObservation, expanded: Path) -> None:
|
|
1060
|
+
"""Verify or create ONE component of the fallback's path-string walk,
|
|
1061
|
+
at *accumulated* (the full path up to and including this component).
|
|
1062
|
+
|
|
1063
|
+
Unlike the POSIX path (:func:`_open_posix`, :func:`_create_absent_component`),
|
|
1064
|
+
there is no ``dir_fd`` to hold: each check here is its own separate
|
|
1065
|
+
``os.stat``/``os.mkdir`` pair against an ACCUMULATED path string. A
|
|
1066
|
+
swap of THIS component after ITS OWN check has already run redirects
|
|
1067
|
+
every SUBSEQUENT component's ``mkdir``/``stat`` for the REMAINDER of
|
|
1068
|
+
the walk — not merely "the next syscall on this same component" — since
|
|
1069
|
+
every later component's accumulated path is built by extending this
|
|
1070
|
+
one's string. See the module docstring's WINDOWS section.
|
|
1071
|
+
"""
|
|
1072
|
+
name = comp.name
|
|
1073
|
+
if comp.identity is None:
|
|
1074
|
+
try:
|
|
1075
|
+
os.mkdir(os.fspath(accumulated))
|
|
1076
|
+
except FileExistsError:
|
|
1077
|
+
pass # benign concurrent create OR an attacker's plant -- the check below decides
|
|
1078
|
+
except OSError as exc:
|
|
1079
|
+
raise DestinationDirectoryError(f"Could not create download destination {expanded}: {exc}") from exc
|
|
1080
|
+
try:
|
|
1081
|
+
info = os.stat(os.fspath(accumulated), follow_symlinks=False)
|
|
1082
|
+
except OSError as exc:
|
|
1083
|
+
raise DestinationDirectoryError(f"Could not open download destination {expanded}: {exc}") from exc
|
|
1084
|
+
# Observed ABSENT: anything other than a plain, non-reparse
|
|
1085
|
+
# directory here now is provably a race, since the observation
|
|
1086
|
+
# already proved nothing was here when the operation began.
|
|
1087
|
+
if stat_module.S_ISLNK(info.st_mode) or _is_reparse_point(info):
|
|
1088
|
+
raise DestinationDirectoryError(
|
|
1089
|
+
f"Refusing to use {expanded} as a download destination: {name!r} was replaced with a "
|
|
1090
|
+
"symlink or junction after it was validated as missing."
|
|
1091
|
+
)
|
|
1092
|
+
if not stat_module.S_ISDIR(info.st_mode):
|
|
1093
|
+
raise DestinationDirectoryError(f"{expanded}: {name!r} exists and is not a directory.")
|
|
1094
|
+
return
|
|
1095
|
+
|
|
1096
|
+
try:
|
|
1097
|
+
info = os.stat(os.fspath(accumulated), follow_symlinks=True)
|
|
1098
|
+
except OSError as exc:
|
|
1099
|
+
raise DestinationDirectoryError(
|
|
1100
|
+
f"Refusing to use {expanded} as a download destination: {name!r} changed since it was "
|
|
1101
|
+
f"validated ({exc})."
|
|
1102
|
+
) from exc
|
|
1103
|
+
if not stat_module.S_ISDIR(info.st_mode):
|
|
1104
|
+
raise DestinationDirectoryError(
|
|
1105
|
+
f"Refusing to use {expanded} as a download destination: {name!r} is not a directory."
|
|
1106
|
+
)
|
|
1107
|
+
identity = (info.st_dev, info.st_ino)
|
|
1108
|
+
if identity != comp.identity:
|
|
1109
|
+
raise DestinationDirectoryError(
|
|
1110
|
+
f"Refusing to use {expanded} as a download destination: {name!r} was replaced with a "
|
|
1111
|
+
"different directory after it was validated."
|
|
1112
|
+
)
|
|
1113
|
+
|
|
1114
|
+
|
|
1115
|
+
def _open_fallback(observation: DestinationObservation) -> _PathIdentityDestinationDirectory:
|
|
1116
|
+
expanded = observation.expanded
|
|
1117
|
+
assert observation.components is not None
|
|
1118
|
+
accumulated = Path(expanded.parts[0]) if observation.is_absolute else Path(".")
|
|
1119
|
+
for comp in observation.components:
|
|
1120
|
+
accumulated = accumulated / comp.name
|
|
1121
|
+
_open_fallback_component(accumulated, comp, expanded)
|
|
1122
|
+
|
|
1123
|
+
# Every component, ancestors included, is now verified/created and
|
|
1124
|
+
# matches the pre-fetch observation. Capture the FINAL entry's current
|
|
1125
|
+
# identity (and whether it is itself a symlink, which decides how
|
|
1126
|
+
# `_PathIdentityDestinationDirectory`'s ONGOING re-checks behave) to
|
|
1127
|
+
# bind the returned handle to — this is the earliest point after the
|
|
1128
|
+
# fetch this call can check it, mirroring what the POSIX fd already
|
|
1129
|
+
# identifies at this same point in the walk.
|
|
1130
|
+
try:
|
|
1131
|
+
leaf_lstat = os.stat(os.fspath(expanded), follow_symlinks=False)
|
|
1132
|
+
except OSError as exc:
|
|
1133
|
+
raise DestinationDirectoryError(f"Could not open download destination {expanded}: {exc}") from exc
|
|
1134
|
+
originally_symlink = stat_module.S_ISLNK(leaf_lstat.st_mode)
|
|
1135
|
+
try:
|
|
1136
|
+
info = os.stat(os.fspath(expanded), follow_symlinks=True)
|
|
1137
|
+
except OSError as exc:
|
|
1138
|
+
raise DestinationDirectoryError(f"Could not open download destination {expanded}: {exc}") from exc
|
|
1139
|
+
if not stat_module.S_ISDIR(info.st_mode):
|
|
1140
|
+
raise DestinationDirectoryError(f"Refusing to use {expanded} as a download destination: it is not a directory.")
|
|
1141
|
+
identity = (info.st_dev, info.st_ino)
|
|
1142
|
+
_refuse_degenerate_identity(identity, expanded)
|
|
1143
|
+
return _PathIdentityDestinationDirectory(expanded, identity, originally_symlink=originally_symlink)
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def open_destination_directory(observation: DestinationObservation) -> DestinationDirectory:
|
|
1147
|
+
"""Open the destination :func:`observe_destination` snapshotted,
|
|
1148
|
+
honoring exactly what it recorded — there is no separate ``directory``
|
|
1149
|
+
argument to (mis)match against ``observation`` here: the observation
|
|
1150
|
+
already carries the expanded path (``observation.expanded``), and a
|
|
1151
|
+
second, independently-suppliable path parameter would only create a
|
|
1152
|
+
caller-bug opportunity for the two to diverge with no attacker
|
|
1153
|
+
involved. Call this immediately before the write it will hold — once
|
|
1154
|
+
bytes are ready to land, with no further network I/O between this call
|
|
1155
|
+
and :meth:`DestinationDirectory.write_atomic`.
|
|
1156
|
+
|
|
1157
|
+
A pre-existing, empty (or non-existing) real directory opens exactly as
|
|
1158
|
+
before; a missing one — including every missing ancestor a nested
|
|
1159
|
+
``-o a/b/c`` needs — is created; see the module docstring for exactly
|
|
1160
|
+
which race this closes on each platform.
|
|
1161
|
+
|
|
1162
|
+
Raises :class:`DestinationDirectoryError` outright if ``observation``
|
|
1163
|
+
was already used for a prior call, or was already closed — this
|
|
1164
|
+
function may be called AT MOST ONCE per :func:`observe_destination`
|
|
1165
|
+
result; see :meth:`DestinationObservation._take_for_open`.
|
|
1166
|
+
|
|
1167
|
+
Ownership of any descriptor ``observation`` was holding transfers to
|
|
1168
|
+
the returned handle on SUCCESS. On a RAISED exception, this call (via
|
|
1169
|
+
``_open_posix``/``_open_fallback``) has already closed whatever it was
|
|
1170
|
+
holding at the point of failure — it does not leak. Callers must still
|
|
1171
|
+
call ``observation.close()`` afterward in a ``finally`` regardless,
|
|
1172
|
+
because a failure can also occur BEFORE this function is ever called
|
|
1173
|
+
(a digest mismatch, a failed fetch) — that is the path this function
|
|
1174
|
+
cannot clean up after, since it never ran; ``observation.close()`` is
|
|
1175
|
+
what covers it, and is always a safe no-op once ownership has already
|
|
1176
|
+
transferred here.
|
|
1177
|
+
"""
|
|
1178
|
+
observation._take_for_open()
|
|
1179
|
+
if SUPPORTS_DIR_FD:
|
|
1180
|
+
return _open_posix(observation)
|
|
1181
|
+
return _open_fallback(observation)
|