autoforge-cli 0.1.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- autoforge/__init__.py +5 -0
- autoforge/agent_worktree.py +653 -0
- autoforge/certificate.py +495 -0
- autoforge/child_env.py +871 -0
- autoforge/claims.py +726 -0
- autoforge/claude_stream.py +501 -0
- autoforge/cli.py +2884 -0
- autoforge/config.py +2178 -0
- autoforge/config_fingerprint.py +374 -0
- autoforge/doctor.py +1551 -0
- autoforge/doctor_sandbox.py +691 -0
- autoforge/effect_ops.py +1272 -0
- autoforge/effects.py +3210 -0
- autoforge/engine.py +16637 -0
- autoforge/epic_commands.py +1486 -0
- autoforge/epic_graph.py +852 -0
- autoforge/errors.py +536 -0
- autoforge/evidence.py +1722 -0
- autoforge/executor.py +1601 -0
- autoforge/executor_duplex.py +985 -0
- autoforge/executor_supervised.py +144 -0
- autoforge/git_transport.py +970 -0
- autoforge/github.py +3402 -0
- autoforge/interaction.py +1027 -0
- autoforge/local_workspace.py +1253 -0
- autoforge/locking.py +1050 -0
- autoforge/loop_detect.py +660 -0
- autoforge/loop_guard.py +404 -0
- autoforge/materialize.py +722 -0
- autoforge/ownership.py +240 -0
- autoforge/pi_rpc.py +713 -0
- autoforge/premerge.py +181 -0
- autoforge/profiles.py +106 -0
- autoforge/progress.py +392 -0
- autoforge/prompts/__init__.py +173 -0
- autoforge/prompts/acceptance_actions.md +39 -0
- autoforge/prompts/analyze_execute.md +128 -0
- autoforge/prompts/common.md +132 -0
- autoforge/prompts/correction.md +12 -0
- autoforge/prompts/fix.md +252 -0
- autoforge/prompts/local_analyze_execute.md +69 -0
- autoforge/prompts/local_common.md +149 -0
- autoforge/prompts/local_fix.md +96 -0
- autoforge/prompts/local_review.md +103 -0
- autoforge/prompts/operator_acceptance.md +61 -0
- autoforge/prompts/replan_reexecute.md +991 -0
- autoforge/prompts/review.md +312 -0
- autoforge/prompts/update_epic.md +120 -0
- autoforge/prompts/update_epic_rerequest.md +85 -0
- autoforge/prompts/validation_evidence.md +26 -0
- autoforge/prompts/validation_failure.md +22 -0
- autoforge/prompts/worktree_state.md +45 -0
- autoforge/providers.py +2893 -0
- autoforge/read_context.py +389 -0
- autoforge/redaction.py +408 -0
- autoforge/replan.py +208 -0
- autoforge/replan_txn.py +1930 -0
- autoforge/result_parser.py +2234 -0
- autoforge/roadmap.py +151 -0
- autoforge/run_contract.py +408 -0
- autoforge/run_namespace.py +839 -0
- autoforge/runlog.py +764 -0
- autoforge/safefs.py +1717 -0
- autoforge/sandbox.py +1313 -0
- autoforge/sandbox_bwrap.py +815 -0
- autoforge/sandbox_pin.py +170 -0
- autoforge/sandbox_policy.py +1107 -0
- autoforge/scheduler.py +1143 -0
- autoforge/scheduler_state.py +2316 -0
- autoforge/scheduler_stop.py +421 -0
- autoforge/sdk_policy.py +988 -0
- autoforge/state.py +2417 -0
- autoforge/transitions.py +388 -0
- autoforge/validation.py +324 -0
- autoforge/worker.py +568 -0
- autoforge_cli-0.1.0a0.dist-info/METADATA +525 -0
- autoforge_cli-0.1.0a0.dist-info/RECORD +81 -0
- autoforge_cli-0.1.0a0.dist-info/WHEEL +5 -0
- autoforge_cli-0.1.0a0.dist-info/entry_points.txt +2 -0
- autoforge_cli-0.1.0a0.dist-info/licenses/LICENSE +21 -0
- autoforge_cli-0.1.0a0.dist-info/top_level.txt +1 -0
autoforge/safefs.py
ADDED
|
@@ -0,0 +1,1717 @@
|
|
|
1
|
+
"""One filesystem capability boundary for every write the controller performs.
|
|
2
|
+
|
|
3
|
+
Why this module exists
|
|
4
|
+
----------------------
|
|
5
|
+
|
|
6
|
+
AutoForge runs coding and review agents as *the same operating-system user*,
|
|
7
|
+
against the same checkout, with no sandbox. Every artifact the controller
|
|
8
|
+
writes therefore lands in a directory tree another process can rearrange
|
|
9
|
+
between one syscall and the next. The historical failure mode was not that
|
|
10
|
+
any single check was wrong, it was that each write site grew its *own* set of
|
|
11
|
+
checks: one module refused a symlinked final component, another refused a
|
|
12
|
+
FIFO, a third remembered hard links, none of them covered a *parent*
|
|
13
|
+
directory that had been replaced by a symbolic link after the check ran.
|
|
14
|
+
That is an open-ended matrix (entry kind x path position x operation) and it
|
|
15
|
+
cannot be closed by adding cases to it.
|
|
16
|
+
|
|
17
|
+
The closed formulation is a *capability*: a directory is opened once, and
|
|
18
|
+
from then on the only way to name anything beneath it is relative to that
|
|
19
|
+
open descriptor.
|
|
20
|
+
|
|
21
|
+
``SafeRoot`` holds an open descriptor on a directory. Every operation walks
|
|
22
|
+
to the target one component at a time with ``O_DIRECTORY | O_NOFOLLOW`` and
|
|
23
|
+
``dir_fd=``, so:
|
|
24
|
+
|
|
25
|
+
* no component of the path -- not the last one, not any parent -- can be a
|
|
26
|
+
symbolic link;
|
|
27
|
+
* renaming or replacing a directory after the walk started cannot redirect
|
|
28
|
+
the write, because the descriptors already resolved are the destination;
|
|
29
|
+
* the final open adds ``O_NOFOLLOW`` (a link fails the open), ``O_NONBLOCK``
|
|
30
|
+
and ``O_NOCTTY`` (a FIFO or a device cannot stall or steal a terminal), and
|
|
31
|
+
an ``fstat`` on the descriptor rejects every non-regular kind;
|
|
32
|
+
* a write open requires ``st_nlink == 1``: more is a hard link, a second
|
|
33
|
+
name for an ordinary file that passes every other test, and none is an
|
|
34
|
+
inode unlinked between the open and the inspection (or a filesystem that
|
|
35
|
+
reports no link count), which a write would reach and report success;
|
|
36
|
+
* ``O_TRUNC`` is never handed to ``os.open`` -- the kernel applies it during
|
|
37
|
+
the open, which is before the descriptor can be inspected -- so a file is
|
|
38
|
+
opened intact, validated, and only then truncated through ``ftruncate`` on
|
|
39
|
+
the very descriptor that was validated.
|
|
40
|
+
|
|
41
|
+
Whole-file writes do not truncate at all: they write and fsync a fresh
|
|
42
|
+
temporary inode in the target's own directory and ``os.replace`` it over the
|
|
43
|
+
target (or, for an exclusive create, ``link(2)`` it to the final name) with
|
|
44
|
+
both sides named by descriptor. That replaces a *name*, never an inode, so
|
|
45
|
+
a hard link planted at the target survives untouched, and a reader either
|
|
46
|
+
sees the whole old file or the whole new one. On Linux the temporary is an
|
|
47
|
+
``O_TMPFILE`` inode: it has *no name at all* while the bytes are written and
|
|
48
|
+
receives its first one when it is published, through the process's own
|
|
49
|
+
``/proc/self/fd`` entry. Where that is unavailable (macOS, a filesystem
|
|
50
|
+
without ``O_TMPFILE``, no procfs) the temporary is created with
|
|
51
|
+
``O_CREAT | O_EXCL`` instead. Either way the inode is one this process
|
|
52
|
+
created, so the bytes can only ever land on something the controller made --
|
|
53
|
+
but a named temporary sits in a directory another process can list, and that
|
|
54
|
+
process can give the inode a second name (``link(2)``) between the create and
|
|
55
|
+
the write. The descriptor is therefore re-inspected after the write and
|
|
56
|
+
before the publish, on both paths: an inode that has acquired a name it was
|
|
57
|
+
not given by the controller is emptied through the still-open descriptor,
|
|
58
|
+
unlinked and reported, never renamed or linked over the target.
|
|
59
|
+
|
|
60
|
+
The one write that is not a whole-file replacement is the append
|
|
61
|
+
(``open_append`` then ``append_to``): a journal line is written through an
|
|
62
|
+
inode opened with ``O_APPEND`` and inspected as a regular file with exactly
|
|
63
|
+
one name, so the cost of an append is the line rather than the file. The
|
|
64
|
+
descriptor is *held* between the open and the append, for as long as the
|
|
65
|
+
caller likes (the run logger holds it across an agent invocation), and the
|
|
66
|
+
bytes go through that descriptor and no other: a held descriptor keeps its
|
|
67
|
+
inode allocated, so no rename, link, unlink or inode-number reuse by another
|
|
68
|
+
process can make the append reach an inode other than the one that was
|
|
69
|
+
inspected. Immediately before the write the append proves again, without
|
|
70
|
+
reading a byte, that the name still resolves under the root to that inode,
|
|
71
|
+
that the inode still has one name, and that it is within the caller's size
|
|
72
|
+
limit; a file renamed over the name, a link given to the inode, or the name
|
|
73
|
+
unlinked or moved away in between is refused, and nothing is written to
|
|
74
|
+
either file.
|
|
75
|
+
|
|
76
|
+
What this does and does not guarantee
|
|
77
|
+
-------------------------------------
|
|
78
|
+
|
|
79
|
+
Guaranteed: a controller write always lands on an inode the controller itself
|
|
80
|
+
created (or opened as a regular file with one name), reached through the
|
|
81
|
+
directory descriptor that was opened as the root, or it fails; and a name the
|
|
82
|
+
controller *publishes* -- the final rename or link -- names an inode that had
|
|
83
|
+
exactly one name at the last inspection before the publish. No sequence of
|
|
84
|
+
symlink, hard link, FIFO, device, directory-replacement or rename operations
|
|
85
|
+
performed by another process under the same user can make a controller write
|
|
86
|
+
reach an inode other than that one: never an object the controller did not
|
|
87
|
+
create or inspect, and never through a name it did not resolve descriptor by
|
|
88
|
+
descriptor. This is a property of descriptor-relative addressing, not of a
|
|
89
|
+
list of rejected shapes, so a shape nobody has thought of yet is covered too.
|
|
90
|
+
The guarantee is about the *inode*; where that inode's name is by the time
|
|
91
|
+
the bytes land is the next paragraph.
|
|
92
|
+
|
|
93
|
+
Not guaranteed everywhere: that a same-user process cannot *observe* a
|
|
94
|
+
controller write through a name it planted. On the named-temporary path a
|
|
95
|
+
temporary can be hard-linked out of the root in the window between its
|
|
96
|
+
creation and the write, and the bytes reach that link before the post-write
|
|
97
|
+
inspection detects it; the write is then refused and the inode emptied, but
|
|
98
|
+
the other process may already have read it. Every byte the controller
|
|
99
|
+
writes is already readable by that process (same user, same files), so this
|
|
100
|
+
discloses nothing, and it can never make a controller write *replace*
|
|
101
|
+
anything: the inode was created by this process, empty, an instant earlier.
|
|
102
|
+
On the Linux path there is no name to link while the bytes are written. An
|
|
103
|
+
exclusive create has no window at all: the inode's first name is the final
|
|
104
|
+
one. A whole-file replacement must still pass through a temporary name,
|
|
105
|
+
because ``linkat`` cannot replace, and holds it for the two syscalls between
|
|
106
|
+
the link and the rename with the re-inspection in between; a link planted
|
|
107
|
+
there finds the bytes complete and is detected before the rename, which is
|
|
108
|
+
the same observe-but-never-replace outcome as the named path in a window
|
|
109
|
+
that no longer spans the write. What remains on every platform, and is
|
|
110
|
+
inherent, is that a same-user process which can open this process's
|
|
111
|
+
``/proc/<pid>/fd`` (that is a ptrace-scope question, not a filesystem one)
|
|
112
|
+
can name the unnamed inode the way the controller itself does; that is why
|
|
113
|
+
the pre-publish inspection requires *zero* names on the Linux path, and why
|
|
114
|
+
such a temporary is emptied and refused exactly like a named one. The
|
|
115
|
+
append's window is one syscall wide and has
|
|
116
|
+
the same shape: between the re-inspection that immediately precedes the
|
|
117
|
+
``write(2)`` and the write itself, the other process can still rename the
|
|
118
|
+
journal's inode out of the root (or link it out and unlink the root name),
|
|
119
|
+
and the line then lands in the controller's own journal under its new name
|
|
120
|
+
with the append reporting success; or unlink the name outright, and the
|
|
121
|
+
line is written into an inode nothing names and lost with it. That is the
|
|
122
|
+
controller's *own* bytes moved or dropped by a process that could already
|
|
123
|
+
read, copy or delete every one of them. What it can never be is a write
|
|
124
|
+
into a foreign file: the bytes go through the descriptor that was
|
|
125
|
+
inspected, so a file put at the journal's name -- renamed there, linked
|
|
126
|
+
there, planted after an unlink -- receives nothing in any window, and is
|
|
127
|
+
refused by the re-inspection in every window but that one syscall. The
|
|
128
|
+
move itself is not checked for beyond that, since it can follow any check;
|
|
129
|
+
it is stated here and pinned by a test. ``O_TMPFILE`` does not help the
|
|
130
|
+
append, which must extend a named file.
|
|
131
|
+
|
|
132
|
+
Also not guaranteed: the *initial* resolution of the root path itself.
|
|
133
|
+
``SafeRoot.open`` resolves an ordinary pathname, and a process that can
|
|
134
|
+
redirect that pathname in the instant before the open can point the
|
|
135
|
+
controller at a different directory. That is equivalent to being able to
|
|
136
|
+
redirect the repository checkout itself and no pathname-based defence can
|
|
137
|
+
close it; it is stated here rather than papered over. Everything *below* the
|
|
138
|
+
root is closed.
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
from __future__ import annotations
|
|
142
|
+
|
|
143
|
+
import errno
|
|
144
|
+
import os
|
|
145
|
+
import secrets
|
|
146
|
+
import stat
|
|
147
|
+
from collections.abc import Iterator
|
|
148
|
+
from dataclasses import dataclass
|
|
149
|
+
from pathlib import Path
|
|
150
|
+
from typing import BinaryIO
|
|
151
|
+
|
|
152
|
+
from .errors import StateError
|
|
153
|
+
|
|
154
|
+
# Absent on Windows; 0 makes the flag a no-op rather than an AttributeError.
|
|
155
|
+
_O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
|
|
156
|
+
_O_NONBLOCK = getattr(os, "O_NONBLOCK", 0)
|
|
157
|
+
_O_NOCTTY = getattr(os, "O_NOCTTY", 0)
|
|
158
|
+
_O_CLOEXEC = getattr(os, "O_CLOEXEC", 0)
|
|
159
|
+
_O_DIRECTORY = getattr(os, "O_DIRECTORY", 0)
|
|
160
|
+
_O_ACCMODE = getattr(os, "O_ACCMODE", 3)
|
|
161
|
+
|
|
162
|
+
_DIR_OPEN_FLAGS = os.O_RDONLY | _O_DIRECTORY | _O_NOFOLLOW | _O_CLOEXEC
|
|
163
|
+
|
|
164
|
+
#: Linux only: an inode created with no directory entry, which receives its
|
|
165
|
+
#: first name when it is published (``SafeRoot._unnamed_tmp_at``). 0 where
|
|
166
|
+
#: the platform has no such thing, and the named temporary is used instead.
|
|
167
|
+
_O_TMPFILE = getattr(os, "O_TMPFILE", 0)
|
|
168
|
+
|
|
169
|
+
#: Directory-relative syscalls this module cannot work without. They exist on
|
|
170
|
+
#: Linux and modern BSD/macOS; refusing early beats silently degrading to
|
|
171
|
+
#: pathname operations that do not hold the invariant.
|
|
172
|
+
_REQUIRED_DIR_FD = ("open", "mkdir", "rename", "unlink", "stat", "link", "readlink", "symlink")
|
|
173
|
+
|
|
174
|
+
#: What ``link(2)`` reports where no hard link can be made to the entry: vfat
|
|
175
|
+
#: and exFAT say ``EPERM`` (as does ``fs.protected_hardlinks`` for a file the
|
|
176
|
+
#: caller neither owns nor may write), several FUSE, SMB/CIFS and overlay
|
|
177
|
+
#: filesystems ``ENOTSUP`` / ``EOPNOTSUPP``, and a FUSE daemon without the
|
|
178
|
+
#: operation ``ENOSYS``. Every one is the same fact for the caller: this
|
|
179
|
+
#: reservation cannot be made here, and another kind must be.
|
|
180
|
+
_NO_HARD_LINKS = frozenset(
|
|
181
|
+
{
|
|
182
|
+
errno.EPERM,
|
|
183
|
+
errno.ENOSYS,
|
|
184
|
+
getattr(errno, "ENOTSUP", errno.EPERM),
|
|
185
|
+
getattr(errno, "EOPNOTSUPP", errno.EPERM),
|
|
186
|
+
}
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
#: What ``fchmod(2)`` reports on a filesystem that holds no permission bits
|
|
190
|
+
#: of its own -- the same filesystems, refusing for the same reason, with
|
|
191
|
+
#: the same errnos: vfat/exFAT and SMB say ``EPERM``, a FUSE daemon without
|
|
192
|
+
#: the operation ``ENOSYS``, others ``ENOTSUP`` / ``EOPNOTSUPP``. There
|
|
193
|
+
#: the bits of source and copy alike are synthesized by the mount, so the
|
|
194
|
+
#: refusal carries nothing the copy's caller can act on; on a filesystem
|
|
195
|
+
#: that *does* hold mode bits, ``fchmod`` of a file this process just
|
|
196
|
+
#: created and still holds open does not fail with any of these.
|
|
197
|
+
_NO_MODE_BITS = _NO_HARD_LINKS
|
|
198
|
+
|
|
199
|
+
#: Chunk of a streamed copy (:meth:`SafeRoot.copy_entry_exclusive`).
|
|
200
|
+
_COPY_CHUNK = 1 << 20
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def entry_kind(mode: int) -> str | None:
|
|
204
|
+
"""Human name of a non-regular entry kind, or ``None`` for a regular file."""
|
|
205
|
+
if stat.S_ISREG(mode):
|
|
206
|
+
return None
|
|
207
|
+
if stat.S_ISDIR(mode):
|
|
208
|
+
return "directory"
|
|
209
|
+
if stat.S_ISFIFO(mode):
|
|
210
|
+
return "FIFO"
|
|
211
|
+
if stat.S_ISSOCK(mode):
|
|
212
|
+
return "socket"
|
|
213
|
+
if stat.S_ISCHR(mode):
|
|
214
|
+
return "character device"
|
|
215
|
+
if stat.S_ISBLK(mode):
|
|
216
|
+
return "block device"
|
|
217
|
+
if stat.S_ISLNK(mode):
|
|
218
|
+
return "symbolic link"
|
|
219
|
+
return "special file"
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
class UnsafePathError(StateError):
|
|
223
|
+
"""A controller path resolves to something the controller may not write.
|
|
224
|
+
|
|
225
|
+
A subclass of :class:`~autoforge.errors.StateError` so existing callers
|
|
226
|
+
and the CLI's error mapping keep working, but distinguishable for tests
|
|
227
|
+
and for callers that want to report the filesystem cause specifically.
|
|
228
|
+
"""
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _denied(exc: OSError) -> bool:
|
|
232
|
+
"""True when the kernel refused for lack of permission, not for shape."""
|
|
233
|
+
return exc.errno in (errno.EACCES, errno.EPERM)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _denied_text(exc: OSError) -> str:
|
|
237
|
+
"""The kernel's own wording, without the filename it repeats back."""
|
|
238
|
+
return exc.strerror or os.strerror(exc.errno or errno.EACCES)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class HardLinksUnavailable(StateError):
|
|
242
|
+
"""A ``link(2)`` was refused because the filesystem, or the kernel's
|
|
243
|
+
policy for this entry, makes no hard links here.
|
|
244
|
+
|
|
245
|
+
From :meth:`SafeRoot.link` it is not a failure of the caller's operation
|
|
246
|
+
but of one way of performing it: a caller that used ``link(2)`` to
|
|
247
|
+
reserve a name without replacing an entry can make the same reservation
|
|
248
|
+
with :meth:`SafeRoot.copy_entry_exclusive` instead.
|
|
249
|
+
|
|
250
|
+
From :meth:`SafeRoot.create_exclusive` it is final. The new file is
|
|
251
|
+
published by linking a complete, fsynced temporary to its name, and
|
|
252
|
+
there is deliberately no fallback that writes the name directly, since
|
|
253
|
+
that is the half-written file the link exists to exclude (ADR 0001,
|
|
254
|
+
known limitation 8; #120). The type lets the caller say what the
|
|
255
|
+
filesystem lacks instead of reporting a generic failed write.
|
|
256
|
+
|
|
257
|
+
Any other refusal of a link stays a plain
|
|
258
|
+
:class:`~autoforge.errors.StateError`.
|
|
259
|
+
"""
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
#: The one statement of what a filesystem without hard links means for the
|
|
263
|
+
#: controller's own directory, appended by the callers that probe it (the
|
|
264
|
+
#: run-log probe and ``doctor``) to a :class:`HardLinksUnavailable` from
|
|
265
|
+
#: :meth:`SafeRoot.create_exclusive`.
|
|
266
|
+
STATE_DIR_NEEDS_HARD_LINKS = (
|
|
267
|
+
"AutoForge's state directory, run logs included, needs a filesystem with hard links "
|
|
268
|
+
"(vfat/exFAT and some FUSE, SMB/CIFS and overlay mounts have none); point --state-dir "
|
|
269
|
+
"or state_dir in the configuration at a directory on one that has them"
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _create_refusal(exc: OSError, where: str) -> StateError:
|
|
274
|
+
"""What a refused ``link(2)`` that gives a new file its name is reported as.
|
|
275
|
+
|
|
276
|
+
A filesystem without hard links (:data:`_NO_HARD_LINKS`) is
|
|
277
|
+
:class:`HardLinksUnavailable` with the reason no other way is tried;
|
|
278
|
+
anything else is the plain failure it always was.
|
|
279
|
+
"""
|
|
280
|
+
if exc.errno in _NO_HARD_LINKS:
|
|
281
|
+
return HardLinksUnavailable(
|
|
282
|
+
f"cannot create {where}: this filesystem makes no hard link here ({exc}). A new "
|
|
283
|
+
"file is given its name only by link(2), once its bytes are complete and "
|
|
284
|
+
"fsynced, so that a crash can never leave it half-written under that name; it is "
|
|
285
|
+
"never written there directly instead"
|
|
286
|
+
)
|
|
287
|
+
return StateError(f"cannot create {where}: {exc}")
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
class ReadLimitExceeded(StateError):
|
|
291
|
+
"""A bounded operation met a file larger than the limit the caller set.
|
|
292
|
+
|
|
293
|
+
Raised by :meth:`SafeRoot.read_bytes` when a read would have to hold
|
|
294
|
+
more than the limit, and by :meth:`SafeRoot.open_append` and
|
|
295
|
+
:meth:`SafeRoot.append_to` when the file is already larger than it: in
|
|
296
|
+
every case the file is refused before it is read or written.
|
|
297
|
+
"""
|
|
298
|
+
|
|
299
|
+
def __init__(self, relpath: str, limit: int) -> None:
|
|
300
|
+
super().__init__(f"{relpath} is larger than {limit} bytes")
|
|
301
|
+
self.relpath = relpath
|
|
302
|
+
self.limit = limit
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
class WalkBudgetExceeded(StateError):
|
|
306
|
+
"""A bounded :meth:`SafeRoot.walk` listed more entries than its budget."""
|
|
307
|
+
|
|
308
|
+
def __init__(self, max_entries: int, where: str) -> None:
|
|
309
|
+
super().__init__(
|
|
310
|
+
f"the walk listed more than {max_entries} directory entries (while listing {where})"
|
|
311
|
+
)
|
|
312
|
+
self.max_entries = max_entries
|
|
313
|
+
self.where = where
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
class UnreadableEntryError(StateError):
|
|
317
|
+
"""An entry exists, is of a kind the controller accepts, and cannot be read.
|
|
318
|
+
|
|
319
|
+
This is *not* an unsafe path: nothing about it could redirect a write. It
|
|
320
|
+
is an access fact -- ``EACCES``/``EPERM`` on an open or a listing -- and
|
|
321
|
+
what it means is the caller's decision, so it is reported as its own type
|
|
322
|
+
carrying the path rather than folded into a generic failure. A controller
|
|
323
|
+
write path treats it as a plain failure; workspace identity translates it
|
|
324
|
+
into a refusal to bind, because a file that cannot be read is a file that
|
|
325
|
+
cannot be proven unchanged.
|
|
326
|
+
"""
|
|
327
|
+
|
|
328
|
+
def __init__(self, path: str, message: str) -> None:
|
|
329
|
+
super().__init__(message)
|
|
330
|
+
self.path = path
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _hard_link_text(nlink: int) -> str:
|
|
334
|
+
return (
|
|
335
|
+
f"a hard link: {nlink} directory entries name this file, so writing here would "
|
|
336
|
+
"alter a file AutoForge did not create"
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _unsafe(where: str, what: str) -> UnsafePathError:
|
|
341
|
+
return UnsafePathError(
|
|
342
|
+
f"refusing to use {where}: it is {what}. AutoForge only reads and writes its own "
|
|
343
|
+
"regular files and directories there; an entry it did not create could redirect a "
|
|
344
|
+
"controller write outside its root or block the controller indefinitely. "
|
|
345
|
+
"Move the entry aside and re-run."
|
|
346
|
+
)
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _empty_and_refuse(fd: int, where: str, what: str) -> UnsafePathError:
|
|
350
|
+
"""Empty the controller's own temporary through ``fd`` and return the
|
|
351
|
+
refusal to publish it, for the caller to raise.
|
|
352
|
+
|
|
353
|
+
Every temporary is inspected once its bytes are complete, and one that
|
|
354
|
+
another process has given a name of its own is not published: it is
|
|
355
|
+
emptied first, so that the other name keeps an empty file and not the
|
|
356
|
+
controller's bytes, and then refused. The emptying is a syscall that can
|
|
357
|
+
fail too, and a failure of it is *this write's* failure -- the bytes
|
|
358
|
+
still sit behind the foreign name -- reported through the controller's
|
|
359
|
+
error type with both facts in it, never as a raw ``OSError`` that the
|
|
360
|
+
CLI's error boundary does not catch.
|
|
361
|
+
"""
|
|
362
|
+
try:
|
|
363
|
+
os.ftruncate(fd, 0)
|
|
364
|
+
except OSError as exc:
|
|
365
|
+
raise StateError(
|
|
366
|
+
f"cannot write {where}: it is {what}, and the controller's bytes could not "
|
|
367
|
+
f"be emptied from that temporary: {exc}"
|
|
368
|
+
) from exc
|
|
369
|
+
return _unsafe(where, what)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def split_relpath(relpath: str) -> tuple[str, ...]:
|
|
373
|
+
"""Validate ``relpath`` as a root-relative path and return its components.
|
|
374
|
+
|
|
375
|
+
Rejects absolute paths, empty components, ``.``, ``..`` and NUL. This is
|
|
376
|
+
the only place a caller-supplied (and therefore possibly state-file- or
|
|
377
|
+
config-supplied) name becomes a sequence of directory-relative lookups,
|
|
378
|
+
so it is the only place the rule has to hold.
|
|
379
|
+
"""
|
|
380
|
+
if not isinstance(relpath, str) or not relpath:
|
|
381
|
+
raise UnsafePathError(f"invalid controller path {relpath!r}: it must be a non-empty string")
|
|
382
|
+
if "\0" in relpath:
|
|
383
|
+
raise UnsafePathError(f"invalid controller path {relpath!r}: it contains a NUL byte")
|
|
384
|
+
if relpath.startswith("/") or (os.sep != "/" and relpath.startswith(os.sep)):
|
|
385
|
+
raise UnsafePathError(
|
|
386
|
+
f"invalid controller path {relpath!r}: it must be relative to the controller root"
|
|
387
|
+
)
|
|
388
|
+
parts = tuple(p for p in relpath.replace(os.sep, "/").split("/") if p != "")
|
|
389
|
+
if not parts:
|
|
390
|
+
raise UnsafePathError(f"invalid controller path {relpath!r}: it names no entry")
|
|
391
|
+
for part in parts:
|
|
392
|
+
if part in (".", ".."):
|
|
393
|
+
raise UnsafePathError(
|
|
394
|
+
f"invalid controller path {relpath!r}: {part!r} is not a name the controller "
|
|
395
|
+
"may traverse; every component must be a plain entry name"
|
|
396
|
+
)
|
|
397
|
+
return parts
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def open_regular_at(
|
|
401
|
+
dir_fd: int,
|
|
402
|
+
name: str,
|
|
403
|
+
flags: int,
|
|
404
|
+
*,
|
|
405
|
+
mode: int = 0o600,
|
|
406
|
+
where: str | None = None,
|
|
407
|
+
) -> int:
|
|
408
|
+
"""``os.open`` beneath ``dir_fd``, restricted to a regular file with one name.
|
|
409
|
+
|
|
410
|
+
Returns a descriptor the caller owns. See the module docstring for why
|
|
411
|
+
``O_TRUNC`` is applied afterwards with ``ftruncate`` rather than passed to
|
|
412
|
+
the open, and why a write open insists on a link count of one.
|
|
413
|
+
"""
|
|
414
|
+
label = where or name
|
|
415
|
+
fd = _open_at(dir_fd, name, flags, mode=mode, label=label)
|
|
416
|
+
try:
|
|
417
|
+
_check_regular_fd(fd, flags, label=label)
|
|
418
|
+
except BaseException:
|
|
419
|
+
os.close(fd)
|
|
420
|
+
raise
|
|
421
|
+
return fd
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _open_at(dir_fd: int, name: str, flags: int, *, mode: int, label: str) -> int:
|
|
425
|
+
"""The open half of :func:`open_regular_at`: the descriptor it returns
|
|
426
|
+
has not been inspected yet.
|
|
427
|
+
|
|
428
|
+
Kept apart so a caller that creates the name (``O_CREAT | O_EXCL``) can
|
|
429
|
+
tell a failed open, which created nothing, from a failed inspection of
|
|
430
|
+
the entry it just created, which it must remove again. A refusal here
|
|
431
|
+
is classified from the errno alone; the ones the kernel can only report
|
|
432
|
+
after the open live in :func:`_check_regular_fd`.
|
|
433
|
+
"""
|
|
434
|
+
open_flags = (flags & ~os.O_TRUNC) | _O_NOFOLLOW | _O_NONBLOCK | _O_NOCTTY | _O_CLOEXEC
|
|
435
|
+
try:
|
|
436
|
+
return os.open(name, open_flags, mode, dir_fd=dir_fd)
|
|
437
|
+
except FileNotFoundError:
|
|
438
|
+
# Absence is the caller's business (see SafeRoot.read_bytes).
|
|
439
|
+
raise
|
|
440
|
+
except FileExistsError:
|
|
441
|
+
# O_EXCL: also the caller's business.
|
|
442
|
+
raise
|
|
443
|
+
except OSError as exc:
|
|
444
|
+
if exc.errno in (errno.ELOOP, errno.EMLINK):
|
|
445
|
+
raise _unsafe(label, "a symbolic link") from exc
|
|
446
|
+
if exc.errno == errno.ENXIO:
|
|
447
|
+
# O_WRONLY|O_NONBLOCK on a FIFO with no reader.
|
|
448
|
+
raise _unsafe(label, "a FIFO") from exc
|
|
449
|
+
if exc.errno == errno.EISDIR:
|
|
450
|
+
# The kernel refuses a writable open before fstat can report it.
|
|
451
|
+
raise _unsafe(label, "a directory, not a regular file") from exc
|
|
452
|
+
if _denied(exc):
|
|
453
|
+
raise UnreadableEntryError(label, f"cannot open {label}: {_denied_text(exc)}") from exc
|
|
454
|
+
raise StateError(f"cannot open {label}: {exc}") from exc
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _check_regular_fd(fd: int, flags: int, *, label: str) -> None:
|
|
458
|
+
"""The inspection half of :func:`open_regular_at`: refuse ``fd`` unless
|
|
459
|
+
it is a regular file that a write open of ``flags`` may use, and apply
|
|
460
|
+
the deferred ``O_TRUNC``. Raises without closing ``fd``; the caller
|
|
461
|
+
owns the descriptor and whatever cleanup its failure calls for."""
|
|
462
|
+
truncate = bool(flags & os.O_TRUNC)
|
|
463
|
+
writing = (flags & _O_ACCMODE) in (os.O_WRONLY, os.O_RDWR)
|
|
464
|
+
st = os.fstat(fd)
|
|
465
|
+
kind = entry_kind(st.st_mode)
|
|
466
|
+
if kind is not None:
|
|
467
|
+
raise _unsafe(label, f"a {kind}, not a regular file")
|
|
468
|
+
# A write open requires exactly one name. More than one is a hard
|
|
469
|
+
# link, and the wording is mode-neutral: a truncating open would
|
|
470
|
+
# replace the shared inode's contents, an appending one would extend
|
|
471
|
+
# them, and either way the change shows at the other name. None at
|
|
472
|
+
# all means the name was unlinked between the open and this
|
|
473
|
+
# inspection (the descriptor still holds the inode, so a write would
|
|
474
|
+
# go into a file nothing names and report success), or that the
|
|
475
|
+
# filesystem reports no link count, in which case a second name can
|
|
476
|
+
# never be ruled out; both are refused rather than written to.
|
|
477
|
+
if writing and st.st_nlink > 1:
|
|
478
|
+
raise _unsafe(label, _hard_link_text(st.st_nlink))
|
|
479
|
+
if writing and st.st_nlink < 1:
|
|
480
|
+
raise _unsafe(
|
|
481
|
+
label,
|
|
482
|
+
"a file no directory entry names: it was unlinked after it was opened, or "
|
|
483
|
+
"this filesystem reports no link count, so a write here could not be "
|
|
484
|
+
"proved to reach a single-named file",
|
|
485
|
+
)
|
|
486
|
+
if truncate:
|
|
487
|
+
os.ftruncate(fd, 0)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def readlink_at(dir_fd: int, name: str) -> str:
|
|
491
|
+
"""Read a symbolic link's target text beneath ``dir_fd``."""
|
|
492
|
+
return os.readlink(name, dir_fd=dir_fd)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
@dataclass
|
|
496
|
+
class WalkEntry:
|
|
497
|
+
"""One entry produced by :meth:`SafeRoot.walk`.
|
|
498
|
+
|
|
499
|
+
``dir_fd`` is the descriptor of the entry's *parent* directory and is
|
|
500
|
+
valid only until the walk advances; use it with :func:`open_regular_at` or
|
|
501
|
+
:func:`readlink_at` while handling the entry, never afterwards. Set
|
|
502
|
+
:attr:`skip` on a directory entry to stop the walk descending into it.
|
|
503
|
+
"""
|
|
504
|
+
|
|
505
|
+
relpath: str
|
|
506
|
+
name: str
|
|
507
|
+
dir_fd: int
|
|
508
|
+
st: os.stat_result
|
|
509
|
+
skip: bool = False
|
|
510
|
+
|
|
511
|
+
@property
|
|
512
|
+
def is_dir(self) -> bool:
|
|
513
|
+
return stat.S_ISDIR(self.st.st_mode)
|
|
514
|
+
|
|
515
|
+
@property
|
|
516
|
+
def is_symlink(self) -> bool:
|
|
517
|
+
return stat.S_ISLNK(self.st.st_mode)
|
|
518
|
+
|
|
519
|
+
@property
|
|
520
|
+
def is_regular(self) -> bool:
|
|
521
|
+
return stat.S_ISREG(self.st.st_mode)
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
class AppendHandle:
|
|
525
|
+
"""A regular file held open for appending, bound to the inode it was proved on.
|
|
526
|
+
|
|
527
|
+
Made by :meth:`SafeRoot.open_append` and written through by
|
|
528
|
+
:meth:`SafeRoot.append_to`. The handle *is* the proof: when it was
|
|
529
|
+
opened, the descriptor was inspected as a single-named regular file at
|
|
530
|
+
``relpath`` under its root, within ``limit``; holding it keeps that inode
|
|
531
|
+
allocated, so its number cannot be reused by a file another process
|
|
532
|
+
creates, and every append through it can reach that inode and no other.
|
|
533
|
+
The instance owns the descriptor and must be closed (it is a context
|
|
534
|
+
manager); dropping it closes it too, as descriptor hygiene rather than
|
|
535
|
+
as anything safety depends on.
|
|
536
|
+
"""
|
|
537
|
+
|
|
538
|
+
__slots__ = ("_fd", "_relpath", "_dev", "_ino", "_limit", "_closed")
|
|
539
|
+
|
|
540
|
+
def __init__(self, fd: int, relpath: str, st: os.stat_result, limit: int | None) -> None:
|
|
541
|
+
self._fd = fd
|
|
542
|
+
self._relpath = relpath
|
|
543
|
+
self._dev = st.st_dev
|
|
544
|
+
self._ino = st.st_ino
|
|
545
|
+
self._limit = limit
|
|
546
|
+
self._closed = False
|
|
547
|
+
|
|
548
|
+
@property
|
|
549
|
+
def fd(self) -> int:
|
|
550
|
+
if self._closed:
|
|
551
|
+
raise StateError(f"append handle on {self._relpath} is already closed")
|
|
552
|
+
return self._fd
|
|
553
|
+
|
|
554
|
+
@property
|
|
555
|
+
def relpath(self) -> str:
|
|
556
|
+
"""The root-relative name the file was opened at."""
|
|
557
|
+
return self._relpath
|
|
558
|
+
|
|
559
|
+
@property
|
|
560
|
+
def identity(self) -> tuple[int, int]:
|
|
561
|
+
"""``(st_dev, st_ino)`` of the inode the handle holds."""
|
|
562
|
+
return (self._dev, self._ino)
|
|
563
|
+
|
|
564
|
+
@property
|
|
565
|
+
def limit(self) -> int | None:
|
|
566
|
+
"""The size past which an append through this handle is refused."""
|
|
567
|
+
return self._limit
|
|
568
|
+
|
|
569
|
+
def close(self) -> None:
|
|
570
|
+
if not self._closed:
|
|
571
|
+
self._closed = True
|
|
572
|
+
os.close(self._fd)
|
|
573
|
+
|
|
574
|
+
def __del__(self) -> None:
|
|
575
|
+
try:
|
|
576
|
+
self.close()
|
|
577
|
+
except Exception: # pragma: no cover - interpreter shutdown
|
|
578
|
+
pass
|
|
579
|
+
|
|
580
|
+
def __enter__(self) -> AppendHandle:
|
|
581
|
+
return self
|
|
582
|
+
|
|
583
|
+
def __exit__(self, *exc: object) -> None:
|
|
584
|
+
self.close()
|
|
585
|
+
|
|
586
|
+
def __repr__(self) -> str: # pragma: no cover - debugging aid
|
|
587
|
+
return f"AppendHandle({self._relpath!r})"
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
class SafeRoot:
|
|
591
|
+
"""An open directory descriptor plus descriptor-relative operations on it.
|
|
592
|
+
|
|
593
|
+
Construct with :meth:`open`. The instance owns a descriptor and must be
|
|
594
|
+
closed (it is a context manager). Every method takes a *root-relative*
|
|
595
|
+
path and reaches it one component at a time; nothing below the root is
|
|
596
|
+
ever addressed by pathname.
|
|
597
|
+
"""
|
|
598
|
+
|
|
599
|
+
__slots__ = ("_fd", "_path", "_dev", "_ino", "_closed")
|
|
600
|
+
|
|
601
|
+
def __init__(self, fd: int, path: Path, st: os.stat_result) -> None:
|
|
602
|
+
self._fd = fd
|
|
603
|
+
self._path = path
|
|
604
|
+
self._dev = st.st_dev
|
|
605
|
+
self._ino = st.st_ino
|
|
606
|
+
self._closed = False
|
|
607
|
+
|
|
608
|
+
# -- construction -------------------------------------------------------
|
|
609
|
+
@classmethod
|
|
610
|
+
def open(cls, path: str | Path, *, create: bool = False) -> SafeRoot:
|
|
611
|
+
"""Open ``path`` as a capability root.
|
|
612
|
+
|
|
613
|
+
``create`` makes the root (and its parents) if absent. The root path
|
|
614
|
+
itself is an ordinary pathname resolution -- see the module docstring
|
|
615
|
+
for exactly what that does and does not promise.
|
|
616
|
+
"""
|
|
617
|
+
missing = [name for name in _REQUIRED_DIR_FD if name not in _supports_dir_fd()]
|
|
618
|
+
if missing:
|
|
619
|
+
raise StateError(
|
|
620
|
+
"this platform does not support directory-relative "
|
|
621
|
+
f"{', '.join(missing)}; AutoForge cannot guarantee that a controller write "
|
|
622
|
+
"stays inside its root here and refuses to run rather than pretend"
|
|
623
|
+
)
|
|
624
|
+
p = Path(path)
|
|
625
|
+
if create:
|
|
626
|
+
_mkdir_tree(p)
|
|
627
|
+
try:
|
|
628
|
+
fd = os.open(p, _DIR_OPEN_FLAGS & ~_O_NOFOLLOW)
|
|
629
|
+
except NotADirectoryError as exc:
|
|
630
|
+
raise _unsafe(str(p), "not a directory") from exc
|
|
631
|
+
except FileNotFoundError:
|
|
632
|
+
raise
|
|
633
|
+
except OSError as exc:
|
|
634
|
+
if exc.errno == errno.ENOTDIR:
|
|
635
|
+
raise _unsafe(str(p), "not a directory") from exc
|
|
636
|
+
raise StateError(f"cannot open controller root {p}: {exc}") from exc
|
|
637
|
+
try:
|
|
638
|
+
st = os.fstat(fd)
|
|
639
|
+
if not stat.S_ISDIR(st.st_mode): # pragma: no cover - O_DIRECTORY covers it
|
|
640
|
+
raise _unsafe(str(p), f"a {entry_kind(st.st_mode)}, not a directory")
|
|
641
|
+
except BaseException:
|
|
642
|
+
os.close(fd)
|
|
643
|
+
raise
|
|
644
|
+
return cls(fd, p, st)
|
|
645
|
+
|
|
646
|
+
def subroot(self, relpath: str, *, create: bool = False) -> SafeRoot:
|
|
647
|
+
"""A :class:`SafeRoot` on a directory beneath this one.
|
|
648
|
+
|
|
649
|
+
Reached by the same component-at-a-time walk, so the sub-root is a
|
|
650
|
+
directory that was actually found inside this root: it cannot be a
|
|
651
|
+
symbolic link, and it stays the directory it was even if its name is
|
|
652
|
+
later reused for something else.
|
|
653
|
+
"""
|
|
654
|
+
parts = split_relpath(relpath)
|
|
655
|
+
if create:
|
|
656
|
+
self.ensure_dir(relpath)
|
|
657
|
+
fd = self._walk(parts, create=False)
|
|
658
|
+
try:
|
|
659
|
+
st = os.fstat(fd)
|
|
660
|
+
except BaseException: # pragma: no cover - the open just succeeded
|
|
661
|
+
os.close(fd)
|
|
662
|
+
raise
|
|
663
|
+
return SafeRoot(fd, self._path / Path(*parts), st)
|
|
664
|
+
|
|
665
|
+
# -- identity -----------------------------------------------------------
|
|
666
|
+
@property
|
|
667
|
+
def path(self) -> Path:
|
|
668
|
+
"""The pathname the root was opened through (for messages only)."""
|
|
669
|
+
return self._path
|
|
670
|
+
|
|
671
|
+
@property
|
|
672
|
+
def fd(self) -> int:
|
|
673
|
+
if self._closed:
|
|
674
|
+
raise StateError(f"controller root {self._path} is already closed")
|
|
675
|
+
return self._fd
|
|
676
|
+
|
|
677
|
+
@property
|
|
678
|
+
def identity(self) -> tuple[int, int]:
|
|
679
|
+
"""``(st_dev, st_ino)`` of the directory this root is bound to."""
|
|
680
|
+
return (self._dev, self._ino)
|
|
681
|
+
|
|
682
|
+
def verify_identity(self, *, role: str = "controller root") -> None:
|
|
683
|
+
"""Fail if the root's pathname no longer names the directory we hold.
|
|
684
|
+
|
|
685
|
+
The controller's writes are safe either way -- they go to the
|
|
686
|
+
descriptor -- but a root that has been renamed or replaced means the
|
|
687
|
+
operator is no longer looking at the files the controller is writing,
|
|
688
|
+
so continuing silently would be the wrong kind of correct. The check
|
|
689
|
+
is by ``lstat``: a symbolic link planted at the pathname is reported
|
|
690
|
+
as what it is rather than followed to wherever it points.
|
|
691
|
+
``role`` names the root in the message (``"state directory"``).
|
|
692
|
+
"""
|
|
693
|
+
if self._closed:
|
|
694
|
+
raise StateError(f"{role} {self._path} is already closed")
|
|
695
|
+
try:
|
|
696
|
+
st = os.lstat(self._path)
|
|
697
|
+
except OSError as exc:
|
|
698
|
+
raise StateError(
|
|
699
|
+
f"{role} {self._path} is no longer readable: {exc}. "
|
|
700
|
+
"It was moved or replaced while the controller was running."
|
|
701
|
+
) from exc
|
|
702
|
+
if stat.S_ISLNK(st.st_mode):
|
|
703
|
+
raise _unsafe(
|
|
704
|
+
f"{role} {self._path}",
|
|
705
|
+
"a symbolic link now, planted while the controller was running",
|
|
706
|
+
)
|
|
707
|
+
if (st.st_dev, st.st_ino) != (self._dev, self._ino):
|
|
708
|
+
raise StateError(
|
|
709
|
+
f"{role} {self._path} was replaced while the controller was running: "
|
|
710
|
+
"it now names a different directory. Refusing to continue against a root that "
|
|
711
|
+
"is no longer the one this run started in."
|
|
712
|
+
)
|
|
713
|
+
|
|
714
|
+
def dup(self) -> SafeRoot:
|
|
715
|
+
"""An independent handle on the same directory (closed separately).
|
|
716
|
+
|
|
717
|
+
For handing the held capability to a component that closes what it
|
|
718
|
+
is given: the copy shares the inode binding, not the lifetime.
|
|
719
|
+
"""
|
|
720
|
+
fd = os.dup(self.fd)
|
|
721
|
+
try:
|
|
722
|
+
st = os.fstat(fd)
|
|
723
|
+
except BaseException: # pragma: no cover - the dup just succeeded
|
|
724
|
+
os.close(fd)
|
|
725
|
+
raise
|
|
726
|
+
return SafeRoot(fd, self._path, st)
|
|
727
|
+
|
|
728
|
+
def close(self) -> None:
|
|
729
|
+
if not self._closed:
|
|
730
|
+
self._closed = True
|
|
731
|
+
os.close(self._fd)
|
|
732
|
+
|
|
733
|
+
def __del__(self) -> None:
|
|
734
|
+
# Descriptor hygiene only: a root dropped without ``close`` must not
|
|
735
|
+
# leak its descriptor for the rest of the process. Nothing about
|
|
736
|
+
# safety depends on finalisation running.
|
|
737
|
+
try:
|
|
738
|
+
self.close()
|
|
739
|
+
except Exception: # pragma: no cover - interpreter shutdown
|
|
740
|
+
pass
|
|
741
|
+
|
|
742
|
+
def __enter__(self) -> SafeRoot:
|
|
743
|
+
return self
|
|
744
|
+
|
|
745
|
+
def __exit__(self, *exc: object) -> None:
|
|
746
|
+
self.close()
|
|
747
|
+
|
|
748
|
+
def __repr__(self) -> str: # pragma: no cover - debugging aid
|
|
749
|
+
return f"SafeRoot({str(self._path)!r})"
|
|
750
|
+
|
|
751
|
+
# -- traversal ----------------------------------------------------------
|
|
752
|
+
def _walk(self, parts: tuple[str, ...], *, create: bool) -> int:
|
|
753
|
+
"""Return a descriptor on the directory named by ``parts``.
|
|
754
|
+
|
|
755
|
+
Every component is opened with ``O_DIRECTORY | O_NOFOLLOW`` relative
|
|
756
|
+
to the previous one. The caller owns the returned descriptor.
|
|
757
|
+
"""
|
|
758
|
+
fd = os.dup(self.fd)
|
|
759
|
+
try:
|
|
760
|
+
for depth, part in enumerate(parts):
|
|
761
|
+
if create:
|
|
762
|
+
made = False
|
|
763
|
+
try:
|
|
764
|
+
os.mkdir(part, 0o700, dir_fd=fd)
|
|
765
|
+
made = True
|
|
766
|
+
except FileExistsError:
|
|
767
|
+
pass
|
|
768
|
+
except OSError as exc:
|
|
769
|
+
where = self._describe(parts[: depth + 1])
|
|
770
|
+
raise StateError(
|
|
771
|
+
f"cannot create controller directory {where}: {exc}"
|
|
772
|
+
) from exc
|
|
773
|
+
if made:
|
|
774
|
+
_fsync_fd(fd)
|
|
775
|
+
try:
|
|
776
|
+
nxt = os.open(part, _DIR_OPEN_FLAGS, dir_fd=fd)
|
|
777
|
+
except FileNotFoundError:
|
|
778
|
+
raise
|
|
779
|
+
except OSError as exc:
|
|
780
|
+
where = self._describe(parts[: depth + 1])
|
|
781
|
+
if exc.errno in (errno.ELOOP, errno.EMLINK, errno.ENOTDIR):
|
|
782
|
+
raise _unsafe(where, _kind_at(fd, part, "not a directory")) from exc
|
|
783
|
+
raise StateError(f"cannot open controller directory {where}: {exc}") from exc
|
|
784
|
+
os.close(fd)
|
|
785
|
+
fd = nxt
|
|
786
|
+
return fd
|
|
787
|
+
except BaseException:
|
|
788
|
+
os.close(fd)
|
|
789
|
+
raise
|
|
790
|
+
|
|
791
|
+
def _describe(self, parts: tuple[str, ...]) -> str:
|
|
792
|
+
return str(self._path / Path(*parts)) if parts else str(self._path)
|
|
793
|
+
|
|
794
|
+
def _parent_of(self, parts: tuple[str, ...], *, create: bool) -> int:
|
|
795
|
+
return self._walk(parts[:-1], create=create)
|
|
796
|
+
|
|
797
|
+
# -- directories --------------------------------------------------------
|
|
798
|
+
def ensure_dir(self, relpath: str) -> None:
|
|
799
|
+
"""Create ``relpath`` and every parent, proving each is a real directory."""
|
|
800
|
+
parts = split_relpath(relpath)
|
|
801
|
+
os.close(self._walk(parts, create=True))
|
|
802
|
+
|
|
803
|
+
# -- reads --------------------------------------------------------------
|
|
804
|
+
def lstat(self, relpath: str) -> os.stat_result | None:
|
|
805
|
+
"""``lstat`` of ``relpath``; ``None`` when it does not exist."""
|
|
806
|
+
parts = split_relpath(relpath)
|
|
807
|
+
try:
|
|
808
|
+
parent = self._parent_of(parts, create=False)
|
|
809
|
+
except FileNotFoundError:
|
|
810
|
+
return None
|
|
811
|
+
try:
|
|
812
|
+
return os.lstat(parts[-1], dir_fd=parent)
|
|
813
|
+
except FileNotFoundError:
|
|
814
|
+
return None
|
|
815
|
+
except OSError as exc:
|
|
816
|
+
raise StateError(f"cannot inspect {self._describe(parts)}: {exc}") from exc
|
|
817
|
+
finally:
|
|
818
|
+
os.close(parent)
|
|
819
|
+
|
|
820
|
+
def read_bytes(self, relpath: str, *, limit: int | None = None) -> bytes | None:
|
|
821
|
+
"""Read ``relpath``; ``None`` when it (or a parent) does not exist.
|
|
822
|
+
|
|
823
|
+
With a ``limit`` the read stops one byte past it and raises
|
|
824
|
+
:class:`ReadLimitExceeded`: a file the caller has decided is too
|
|
825
|
+
large to hold is never materialised, whatever ``st_size`` claimed
|
|
826
|
+
before the file was opened.
|
|
827
|
+
"""
|
|
828
|
+
parts = split_relpath(relpath)
|
|
829
|
+
try:
|
|
830
|
+
parent = self._parent_of(parts, create=False)
|
|
831
|
+
except FileNotFoundError:
|
|
832
|
+
return None
|
|
833
|
+
try:
|
|
834
|
+
fd = open_regular_at(parent, parts[-1], os.O_RDONLY, where=self._describe(parts))
|
|
835
|
+
except FileNotFoundError:
|
|
836
|
+
return None
|
|
837
|
+
finally:
|
|
838
|
+
os.close(parent)
|
|
839
|
+
with os.fdopen(fd, "rb") as fh:
|
|
840
|
+
return _read_within(fh, relpath, limit)
|
|
841
|
+
|
|
842
|
+
def read_text(self, relpath: str, *, limit: int | None = None) -> str | None:
|
|
843
|
+
data = self.read_bytes(relpath, limit=limit)
|
|
844
|
+
if data is None:
|
|
845
|
+
return None
|
|
846
|
+
return data.decode("utf-8", errors="replace")
|
|
847
|
+
|
|
848
|
+
# -- writes -------------------------------------------------------------
|
|
849
|
+
def write_bytes(self, relpath: str, data: bytes, *, mode: int = 0o600) -> None:
|
|
850
|
+
"""Atomically make ``relpath`` hold exactly ``data``.
|
|
851
|
+
|
|
852
|
+
A fresh temporary is created in the target's own directory, written,
|
|
853
|
+
fsynced and renamed over the target, with both sides named relative to
|
|
854
|
+
the same directory descriptor. Nothing is ever truncated in place, so
|
|
855
|
+
an existing hard link, and any reader of the old contents, is left
|
|
856
|
+
intact.
|
|
857
|
+
"""
|
|
858
|
+
parts = split_relpath(relpath)
|
|
859
|
+
parent = self._parent_of(parts, create=True)
|
|
860
|
+
try:
|
|
861
|
+
self._replace_at(parent, parts[-1], data, mode=mode, where=self._describe(parts))
|
|
862
|
+
finally:
|
|
863
|
+
os.close(parent)
|
|
864
|
+
|
|
865
|
+
def write_text(self, relpath: str, text: str, *, mode: int = 0o600) -> None:
|
|
866
|
+
self.write_bytes(relpath, text.encode("utf-8"), mode=mode)
|
|
867
|
+
|
|
868
|
+
def create_exclusive(self, relpath: str, data: bytes, *, mode: int = 0o600) -> None:
|
|
869
|
+
"""Create ``relpath`` holding exactly ``data``, or fail because an
|
|
870
|
+
entry of that name already exists.
|
|
871
|
+
|
|
872
|
+
The name is published only after the bytes are durable: ``data`` is
|
|
873
|
+
written and fsynced into a fresh temporary in the same directory and
|
|
874
|
+
then *linked* to ``relpath``. ``link(2)`` fails with ``EEXIST`` on any
|
|
875
|
+
existing entry -- a symbolic link included -- so the exclusivity is
|
|
876
|
+
the same as ``O_CREAT | O_EXCL`` would give, but a crash can no longer
|
|
877
|
+
leave a half-written file behind under the final name for a later
|
|
878
|
+
invocation to find and trust.
|
|
879
|
+
|
|
880
|
+
That makes hard links a requirement: where the filesystem makes none
|
|
881
|
+
the create is refused with :class:`HardLinksUnavailable`, and nothing
|
|
882
|
+
is left under ``relpath`` or as a temporary.
|
|
883
|
+
"""
|
|
884
|
+
parts = split_relpath(relpath)
|
|
885
|
+
parent = self._parent_of(parts, create=True)
|
|
886
|
+
try:
|
|
887
|
+
self._publish_new_at(parent, parts[-1], data, mode=mode, where=self._describe(parts))
|
|
888
|
+
finally:
|
|
889
|
+
os.close(parent)
|
|
890
|
+
|
|
891
|
+
def open_append(
|
|
892
|
+
self, relpath: str, *, mode: int = 0o600, limit: int | None = None
|
|
893
|
+
) -> AppendHandle:
|
|
894
|
+
"""Open ``relpath`` for appending, creating it if needed, and hold it.
|
|
895
|
+
|
|
896
|
+
The one write that is not a whole-file replacement. Replacing the
|
|
897
|
+
name would mean reading the existing bytes and publishing them again
|
|
898
|
+
with the new ones, which costs the whole file on every append; a
|
|
899
|
+
journal appended to N times would be rewritten N times. The append
|
|
900
|
+
therefore goes through an opened inode: ``O_WRONLY | O_CREAT |
|
|
901
|
+
O_APPEND`` with the same ``O_NOFOLLOW | O_NONBLOCK | O_NOCTTY`` open
|
|
902
|
+
as every other controller open, and the descriptor is inspected
|
|
903
|
+
before it is handed back -- a regular file (no link, FIFO, device or
|
|
904
|
+
directory) with exactly one name, or :class:`UnsafePathError`; with
|
|
905
|
+
a ``limit``, one no larger than it, or :class:`ReadLimitExceeded`.
|
|
906
|
+
Nothing is read, so the cost is the open whatever the file's size.
|
|
907
|
+
|
|
908
|
+
The descriptor is returned rather than closed so that the proof
|
|
909
|
+
outlives this call: :meth:`append_to` writes through it, and only
|
|
910
|
+
through it, so a caller that opens the file at one moment (before an
|
|
911
|
+
agent is launched) and appends at another (after it returned) is
|
|
912
|
+
guaranteed that the bytes reach the inode inspected at the first
|
|
913
|
+
moment or no inode at all. The caller owns the handle and closes it;
|
|
914
|
+
until the handle exists, this call does, and a failure anywhere
|
|
915
|
+
after the open -- the size check, or the directory ``fsync`` that
|
|
916
|
+
makes a name this open created durable -- closes the descriptor
|
|
917
|
+
before it raises, so a refused open never leaks it. An ``OSError``
|
|
918
|
+
there is a :class:`~autoforge.errors.StateError`, as one at the
|
|
919
|
+
open itself is; the file, if the open created it, is left where it
|
|
920
|
+
is, since an empty journal is what the next open finds or makes.
|
|
921
|
+
"""
|
|
922
|
+
parts = split_relpath(relpath)
|
|
923
|
+
where = self._describe(parts)
|
|
924
|
+
parent = self._parent_of(parts, create=True)
|
|
925
|
+
try:
|
|
926
|
+
fd = open_regular_at(
|
|
927
|
+
parent, parts[-1], os.O_WRONLY | os.O_CREAT | os.O_APPEND, mode=mode, where=where
|
|
928
|
+
)
|
|
929
|
+
try:
|
|
930
|
+
st = os.fstat(fd)
|
|
931
|
+
if limit is not None and st.st_size > limit:
|
|
932
|
+
raise ReadLimitExceeded(relpath, limit)
|
|
933
|
+
# The name may have been created by this open; its directory
|
|
934
|
+
# entry is made durable the way a published temporary's is.
|
|
935
|
+
_fsync_fd(parent)
|
|
936
|
+
except OSError as exc:
|
|
937
|
+
os.close(fd)
|
|
938
|
+
raise StateError(f"cannot open {where}: {exc}") from exc
|
|
939
|
+
except BaseException:
|
|
940
|
+
os.close(fd)
|
|
941
|
+
raise
|
|
942
|
+
finally:
|
|
943
|
+
os.close(parent)
|
|
944
|
+
return AppendHandle(fd, relpath, st, limit)
|
|
945
|
+
|
|
946
|
+
def append_to(self, handle: AppendHandle, text: str) -> None:
|
|
947
|
+
"""Append ``text`` through ``handle``, once the file is proved still what was opened.
|
|
948
|
+
|
|
949
|
+
The bytes go through the held descriptor, so they can only ever land
|
|
950
|
+
in the inode :meth:`open_append` inspected; that is the guarantee,
|
|
951
|
+
and it holds whatever another process did to the name in between.
|
|
952
|
+
What is proved immediately before the write, each in O(1) and none
|
|
953
|
+
of it reading a byte, is that the file is still the one the caller
|
|
954
|
+
means: the name resolves, under *this* root (walked descriptor by
|
|
955
|
+
descriptor, so a root re-verified since the open is the one that
|
|
956
|
+
answers), to the held inode by ``lstat`` -- a file renamed over the
|
|
957
|
+
name, a symbolic link planted at it, or the name unlinked or moved
|
|
958
|
+
away is refused and nothing is written to whatever is there; the
|
|
959
|
+
inode still has exactly one name -- a hard link given to it since
|
|
960
|
+
the open is refused, as it would have been at the open; and the
|
|
961
|
+
inode is within the handle's limit. Then the line is written and
|
|
962
|
+
fsynced.
|
|
963
|
+
|
|
964
|
+
What remains is the window between that re-inspection and the
|
|
965
|
+
``write(2)``: a move of the inode in it puts the line in the
|
|
966
|
+
caller's own file under its new name, an unlink loses the line with
|
|
967
|
+
the inode, and the append reports success either way. A file the
|
|
968
|
+
other process put at the name receives nothing in any window
|
|
969
|
+
(module docstring, "Not guaranteed").
|
|
970
|
+
"""
|
|
971
|
+
parts = split_relpath(handle.relpath)
|
|
972
|
+
where = self._describe(parts)
|
|
973
|
+
fd = handle.fd
|
|
974
|
+
gone = UnsafePathError(
|
|
975
|
+
f"refusing to append to {where}: no entry of that name is left, so the file that "
|
|
976
|
+
"was opened and inspected there was unlinked or moved away since. Nothing was "
|
|
977
|
+
"written to it."
|
|
978
|
+
)
|
|
979
|
+
try:
|
|
980
|
+
parent = self._parent_of(parts, create=False)
|
|
981
|
+
except FileNotFoundError:
|
|
982
|
+
raise gone from None
|
|
983
|
+
try:
|
|
984
|
+
try:
|
|
985
|
+
now = os.lstat(parts[-1], dir_fd=parent)
|
|
986
|
+
except FileNotFoundError:
|
|
987
|
+
raise gone from None
|
|
988
|
+
except OSError as exc:
|
|
989
|
+
raise StateError(f"cannot inspect {where}: {exc}") from exc
|
|
990
|
+
finally:
|
|
991
|
+
os.close(parent)
|
|
992
|
+
if (now.st_dev, now.st_ino) != handle.identity:
|
|
993
|
+
kind = entry_kind(now.st_mode) or "regular file"
|
|
994
|
+
raise UnsafePathError(
|
|
995
|
+
f"refusing to append to {where}: the entry at that name is a different {kind} "
|
|
996
|
+
"now, not the file that was opened and inspected there, so it was replaced "
|
|
997
|
+
"since. Neither file was written to; move the entry aside and re-run."
|
|
998
|
+
)
|
|
999
|
+
st = os.fstat(fd)
|
|
1000
|
+
if st.st_nlink > 1:
|
|
1001
|
+
raise _unsafe(where, _hard_link_text(st.st_nlink))
|
|
1002
|
+
if st.st_nlink < 1: # pragma: no cover - the name resolved to it an instant ago
|
|
1003
|
+
raise gone
|
|
1004
|
+
if handle.limit is not None and st.st_size > handle.limit:
|
|
1005
|
+
raise ReadLimitExceeded(handle.relpath, handle.limit)
|
|
1006
|
+
try:
|
|
1007
|
+
view = memoryview(text.encode("utf-8"))
|
|
1008
|
+
while view:
|
|
1009
|
+
view = view[os.write(fd, view) :]
|
|
1010
|
+
os.fsync(fd)
|
|
1011
|
+
except OSError as exc:
|
|
1012
|
+
raise StateError(f"cannot append to {where}: {exc}") from exc
|
|
1013
|
+
|
|
1014
|
+
def unlink(self, relpath: str) -> None:
|
|
1015
|
+
parts = split_relpath(relpath)
|
|
1016
|
+
try:
|
|
1017
|
+
parent = self._parent_of(parts, create=False)
|
|
1018
|
+
except FileNotFoundError:
|
|
1019
|
+
return
|
|
1020
|
+
try:
|
|
1021
|
+
os.unlink(parts[-1], dir_fd=parent)
|
|
1022
|
+
except FileNotFoundError:
|
|
1023
|
+
pass
|
|
1024
|
+
except OSError as exc:
|
|
1025
|
+
raise StateError(f"cannot remove {self._describe(parts)}: {exc}") from exc
|
|
1026
|
+
finally:
|
|
1027
|
+
os.close(parent)
|
|
1028
|
+
|
|
1029
|
+
def link(self, src: str, dst: str) -> None:
|
|
1030
|
+
"""Hard-link ``src`` to ``dst`` within this root, never following a link.
|
|
1031
|
+
|
|
1032
|
+
Used to *reserve* a destination name: ``link`` fails atomically with
|
|
1033
|
+
``EEXIST`` when the name is taken, which a rename would not.
|
|
1034
|
+
``follow_symlinks=False`` links the directory entry itself, so a
|
|
1035
|
+
symbolic link is archived as a link rather than resolved. Where the
|
|
1036
|
+
filesystem makes no hard links (:data:`_NO_HARD_LINKS`) the refusal
|
|
1037
|
+
is :class:`HardLinksUnavailable`, so the caller can make the same
|
|
1038
|
+
reservation with :meth:`copy_entry_exclusive` instead.
|
|
1039
|
+
"""
|
|
1040
|
+
src_parts = split_relpath(src)
|
|
1041
|
+
dst_parts = split_relpath(dst)
|
|
1042
|
+
src_parent = self._parent_of(src_parts, create=False)
|
|
1043
|
+
try:
|
|
1044
|
+
dst_parent = self._parent_of(dst_parts, create=True)
|
|
1045
|
+
try:
|
|
1046
|
+
os.link(
|
|
1047
|
+
src_parts[-1],
|
|
1048
|
+
dst_parts[-1],
|
|
1049
|
+
src_dir_fd=src_parent,
|
|
1050
|
+
dst_dir_fd=dst_parent,
|
|
1051
|
+
follow_symlinks=False,
|
|
1052
|
+
)
|
|
1053
|
+
except FileExistsError:
|
|
1054
|
+
raise
|
|
1055
|
+
except OSError as exc:
|
|
1056
|
+
where = f"{self._describe(src_parts)} to {self._describe(dst_parts)}"
|
|
1057
|
+
if exc.errno in _NO_HARD_LINKS:
|
|
1058
|
+
raise HardLinksUnavailable(
|
|
1059
|
+
f"cannot link {where}: this filesystem makes no hard link here ({exc})"
|
|
1060
|
+
) from exc
|
|
1061
|
+
raise StateError(f"cannot link {where}: {exc}") from exc
|
|
1062
|
+
finally:
|
|
1063
|
+
os.close(dst_parent)
|
|
1064
|
+
finally:
|
|
1065
|
+
os.close(src_parent)
|
|
1066
|
+
|
|
1067
|
+
def copy_entry_exclusive(self, src: str, dst: str) -> None:
|
|
1068
|
+
"""Reproduce the directory entry ``src`` under the new name ``dst``
|
|
1069
|
+
within this root, or fail because an entry of that name already
|
|
1070
|
+
exists: the reservation :meth:`link` makes, for a filesystem where
|
|
1071
|
+
:meth:`link` raises :class:`HardLinksUnavailable`.
|
|
1072
|
+
|
|
1073
|
+
The entry itself is reproduced, never what it points at: a regular
|
|
1074
|
+
file's bytes are streamed into ``dst`` opened ``O_CREAT | O_EXCL``
|
|
1075
|
+
and fsynced, with the source's permission bits restored where the
|
|
1076
|
+
filesystem holds any -- where it holds none and ``fchmod(2)`` says
|
|
1077
|
+
so (``EPERM``, ``ENOSYS``, ``ENOTSUP``/``EOPNOTSUPP``, the same
|
|
1078
|
+
refusals as :meth:`link`'s, :data:`_NO_MODE_BITS`), the copy is
|
|
1079
|
+
made anyway and carries the umask-filtered subset of the source's
|
|
1080
|
+
bits, so it is at most as permissive as the source, never more; a
|
|
1081
|
+
symbolic link is recreated by ``symlink(2)`` with its target text
|
|
1082
|
+
and never followed.
|
|
1083
|
+
Both fail with :class:`FileExistsError` on any existing entry at
|
|
1084
|
+
``dst`` (a symbolic link included), exactly as :meth:`link` does. A
|
|
1085
|
+
FIFO, socket or device has no bytes to copy and is refused; the
|
|
1086
|
+
caller is told to move it by hand. Nothing is ever read through a
|
|
1087
|
+
symbolic link or written anywhere but the fresh entry.
|
|
1088
|
+
|
|
1089
|
+
Unlike a hard link this is not one syscall: the bytes reach ``dst``
|
|
1090
|
+
after its name exists. Every failure after the name is created --
|
|
1091
|
+
the fresh entry failing inspection, the permission bits (for any
|
|
1092
|
+
other errno than the ones above), the bytes,
|
|
1093
|
+
their fsync, or the fsync of the directory that publishes the name
|
|
1094
|
+
-- removes that name again and nothing else, so a failed reservation
|
|
1095
|
+
leaves the directory as it was found. That differs from the
|
|
1096
|
+
controller's own writes, which keep a name whose directory fsync
|
|
1097
|
+
failed (:meth:`_fsync_published`): those bytes are the product,
|
|
1098
|
+
whereas here the source is still in place and a copy whose
|
|
1099
|
+
durability could not be proved is worth nothing to the caller, who
|
|
1100
|
+
must not remove the source on the strength of it. A *crash*
|
|
1101
|
+
mid-copy leaves a partial file under ``dst``, which later exclusive
|
|
1102
|
+
creates will not replace and which the source, still in place, does
|
|
1103
|
+
not depend on.
|
|
1104
|
+
"""
|
|
1105
|
+
src_parts = split_relpath(src)
|
|
1106
|
+
dst_parts = split_relpath(dst)
|
|
1107
|
+
src_parent = self._parent_of(src_parts, create=False)
|
|
1108
|
+
try:
|
|
1109
|
+
dst_parent = self._parent_of(dst_parts, create=True)
|
|
1110
|
+
try:
|
|
1111
|
+
self._copy_entry_at(
|
|
1112
|
+
src_parent,
|
|
1113
|
+
src_parts[-1],
|
|
1114
|
+
dst_parent,
|
|
1115
|
+
dst_parts[-1],
|
|
1116
|
+
where_src=self._describe(src_parts),
|
|
1117
|
+
where_dst=self._describe(dst_parts),
|
|
1118
|
+
)
|
|
1119
|
+
finally:
|
|
1120
|
+
os.close(dst_parent)
|
|
1121
|
+
finally:
|
|
1122
|
+
os.close(src_parent)
|
|
1123
|
+
|
|
1124
|
+
def _copy_entry_at(
|
|
1125
|
+
self,
|
|
1126
|
+
src_parent: int,
|
|
1127
|
+
src_name: str,
|
|
1128
|
+
dst_parent: int,
|
|
1129
|
+
dst_name: str,
|
|
1130
|
+
*,
|
|
1131
|
+
where_src: str,
|
|
1132
|
+
where_dst: str,
|
|
1133
|
+
) -> None:
|
|
1134
|
+
where = f"{where_src} to {where_dst}"
|
|
1135
|
+
try:
|
|
1136
|
+
st = os.lstat(src_name, dir_fd=src_parent)
|
|
1137
|
+
except OSError as exc:
|
|
1138
|
+
raise StateError(f"cannot copy {where}: {exc}") from exc
|
|
1139
|
+
if stat.S_ISLNK(st.st_mode):
|
|
1140
|
+
try:
|
|
1141
|
+
target = os.readlink(src_name, dir_fd=src_parent)
|
|
1142
|
+
os.symlink(target, dst_name, dir_fd=dst_parent)
|
|
1143
|
+
except FileExistsError:
|
|
1144
|
+
raise
|
|
1145
|
+
except OSError as exc:
|
|
1146
|
+
raise StateError(f"cannot copy {where}: {exc}") from exc
|
|
1147
|
+
# From here on the name is this process's; see _copy_bytes_at.
|
|
1148
|
+
try:
|
|
1149
|
+
self._fsync_published(dst_parent, where_dst)
|
|
1150
|
+
except BaseException:
|
|
1151
|
+
_quiet_unlink(dst_parent, dst_name)
|
|
1152
|
+
raise
|
|
1153
|
+
return
|
|
1154
|
+
kind = entry_kind(st.st_mode)
|
|
1155
|
+
if kind is not None:
|
|
1156
|
+
raise StateError(
|
|
1157
|
+
f"cannot copy {where}: it is a {kind}, not a regular file or a symbolic "
|
|
1158
|
+
"link, and has no bytes to copy; move it by hand"
|
|
1159
|
+
)
|
|
1160
|
+
in_fd = open_regular_at(src_parent, src_name, os.O_RDONLY, where=where_src)
|
|
1161
|
+
try:
|
|
1162
|
+
self._copy_bytes_at(
|
|
1163
|
+
in_fd,
|
|
1164
|
+
dst_parent,
|
|
1165
|
+
dst_name,
|
|
1166
|
+
mode=stat.S_IMODE(st.st_mode),
|
|
1167
|
+
where=where,
|
|
1168
|
+
where_dst=where_dst,
|
|
1169
|
+
)
|
|
1170
|
+
finally:
|
|
1171
|
+
os.close(in_fd)
|
|
1172
|
+
|
|
1173
|
+
def _copy_bytes_at(
|
|
1174
|
+
self,
|
|
1175
|
+
in_fd: int,
|
|
1176
|
+
dst_parent: int,
|
|
1177
|
+
dst_name: str,
|
|
1178
|
+
*,
|
|
1179
|
+
mode: int,
|
|
1180
|
+
where: str,
|
|
1181
|
+
where_dst: str,
|
|
1182
|
+
) -> None:
|
|
1183
|
+
"""Create ``dst_name`` exclusively and fill it from ``in_fd``.
|
|
1184
|
+
|
|
1185
|
+
The open is split in two so that a failure *creating* the name,
|
|
1186
|
+
which leaves nothing behind, is told apart from every failure
|
|
1187
|
+
*after* it: the name is this process's from the moment ``O_EXCL``
|
|
1188
|
+
succeeds, and any of those later failures removes it again (and
|
|
1189
|
+
only it: a second name someone else gave the inode meanwhile is
|
|
1190
|
+
theirs) so the directory is left as it was found.
|
|
1191
|
+
"""
|
|
1192
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
1193
|
+
# FileExistsError from O_EXCL is the caller's business, as in
|
|
1194
|
+
# open_regular_at; every other refusal is already a StateError.
|
|
1195
|
+
out_fd = _open_at(dst_parent, dst_name, flags, mode=mode, label=where_dst)
|
|
1196
|
+
try:
|
|
1197
|
+
try:
|
|
1198
|
+
_check_regular_fd(out_fd, flags, label=where_dst)
|
|
1199
|
+
# O_CREAT applied the umask to ``mode``; the source's bits
|
|
1200
|
+
# are what a hard link would have carried, so restore them
|
|
1201
|
+
# where the filesystem holds any. Where it holds none
|
|
1202
|
+
# (_NO_MODE_BITS: the same mounts that have no link(2), which
|
|
1203
|
+
# is why this copy is running at all) the copy carries the
|
|
1204
|
+
# umask-filtered subset of the source's bits, never more,
|
|
1205
|
+
# and the bytes are what the caller came for.
|
|
1206
|
+
try:
|
|
1207
|
+
os.fchmod(out_fd, mode)
|
|
1208
|
+
except OSError as exc:
|
|
1209
|
+
if exc.errno not in _NO_MODE_BITS:
|
|
1210
|
+
raise
|
|
1211
|
+
while chunk := os.read(in_fd, _COPY_CHUNK):
|
|
1212
|
+
view = memoryview(chunk)
|
|
1213
|
+
while view:
|
|
1214
|
+
view = view[os.write(out_fd, view) :]
|
|
1215
|
+
os.fsync(out_fd)
|
|
1216
|
+
except OSError as exc:
|
|
1217
|
+
raise StateError(f"cannot copy {where}: {exc}") from exc
|
|
1218
|
+
finally:
|
|
1219
|
+
os.close(out_fd)
|
|
1220
|
+
self._fsync_published(dst_parent, where_dst)
|
|
1221
|
+
except BaseException:
|
|
1222
|
+
_quiet_unlink(dst_parent, dst_name)
|
|
1223
|
+
raise
|
|
1224
|
+
|
|
1225
|
+
def rename(self, src: str, dst: str) -> None:
|
|
1226
|
+
"""Rename within this root, both sides named relative to a descriptor."""
|
|
1227
|
+
src_parts = split_relpath(src)
|
|
1228
|
+
dst_parts = split_relpath(dst)
|
|
1229
|
+
src_parent = self._parent_of(src_parts, create=False)
|
|
1230
|
+
try:
|
|
1231
|
+
dst_parent = self._parent_of(dst_parts, create=True)
|
|
1232
|
+
try:
|
|
1233
|
+
os.rename(
|
|
1234
|
+
src_parts[-1],
|
|
1235
|
+
dst_parts[-1],
|
|
1236
|
+
src_dir_fd=src_parent,
|
|
1237
|
+
dst_dir_fd=dst_parent,
|
|
1238
|
+
)
|
|
1239
|
+
_fsync_fd(dst_parent)
|
|
1240
|
+
except OSError as exc:
|
|
1241
|
+
raise StateError(
|
|
1242
|
+
f"cannot move {self._describe(src_parts)} to {self._describe(dst_parts)}: {exc}"
|
|
1243
|
+
) from exc
|
|
1244
|
+
finally:
|
|
1245
|
+
os.close(dst_parent)
|
|
1246
|
+
finally:
|
|
1247
|
+
os.close(src_parent)
|
|
1248
|
+
|
|
1249
|
+
def _unnamed_tmp_at(self, parent: int, data: bytes, *, mode: int, where: str) -> int | None:
|
|
1250
|
+
"""Write ``data``, fsynced, into an inode of ``parent``'s filesystem
|
|
1251
|
+
that has no name (Linux ``O_TMPFILE``).
|
|
1252
|
+
|
|
1253
|
+
Returns a descriptor the caller owns and publishes through
|
|
1254
|
+
:meth:`_link_unnamed_at`, or ``None`` where the mechanism is not
|
|
1255
|
+
available, in which case the caller falls back to
|
|
1256
|
+
:meth:`_durable_tmp_at`. Unavailable means: the platform has no
|
|
1257
|
+
``O_TMPFILE`` (macOS); the kernel refuses the open (a filesystem
|
|
1258
|
+
without it reports ``EOPNOTSUPP``, a kernel too old to know the flag
|
|
1259
|
+
sees the ``O_DIRECTORY`` bit it is built on and reports ``EISDIR``,
|
|
1260
|
+
an emulation layer may say ``EINVAL``); or the process's own
|
|
1261
|
+
``/proc/self/fd`` entry does not name the inode just opened (no
|
|
1262
|
+
procfs, or something else mounted there). *Every* failure of the
|
|
1263
|
+
open falls back rather than being classified, because the fallback
|
|
1264
|
+
opens with ``O_CREAT | O_EXCL`` in the same directory an instant
|
|
1265
|
+
later: a real refusal -- permission, space, a read-only mount --
|
|
1266
|
+
recurs there and is reported by the path that has always reported
|
|
1267
|
+
it, so nothing is hidden and the fallback costs one syscall.
|
|
1268
|
+
|
|
1269
|
+
A write failure *after* the open is this write's failure and is
|
|
1270
|
+
raised; the inode has no name, so closing the descriptor is the whole
|
|
1271
|
+
cleanup.
|
|
1272
|
+
"""
|
|
1273
|
+
if not _O_TMPFILE:
|
|
1274
|
+
return None
|
|
1275
|
+
try:
|
|
1276
|
+
fd = os.open(".", os.O_WRONLY | _O_TMPFILE | _O_CLOEXEC, mode, dir_fd=parent)
|
|
1277
|
+
except OSError:
|
|
1278
|
+
return None
|
|
1279
|
+
try:
|
|
1280
|
+
st = os.fstat(fd)
|
|
1281
|
+
try:
|
|
1282
|
+
seen = os.stat(_proc_fd_path(fd))
|
|
1283
|
+
except OSError:
|
|
1284
|
+
seen = None
|
|
1285
|
+
if (
|
|
1286
|
+
seen is None
|
|
1287
|
+
or (seen.st_dev, seen.st_ino) != (st.st_dev, st.st_ino)
|
|
1288
|
+
or entry_kind(st.st_mode) is not None
|
|
1289
|
+
or st.st_nlink != 0
|
|
1290
|
+
):
|
|
1291
|
+
# The publish below needs the procfs entry to denote exactly
|
|
1292
|
+
# this inode; without that proof the named path is used.
|
|
1293
|
+
os.close(fd)
|
|
1294
|
+
return None
|
|
1295
|
+
view = memoryview(data)
|
|
1296
|
+
while view:
|
|
1297
|
+
view = view[os.write(fd, view) :]
|
|
1298
|
+
os.fsync(fd)
|
|
1299
|
+
except OSError as exc:
|
|
1300
|
+
os.close(fd)
|
|
1301
|
+
raise StateError(f"cannot write {where}: {exc}") from exc
|
|
1302
|
+
except BaseException:
|
|
1303
|
+
os.close(fd)
|
|
1304
|
+
raise
|
|
1305
|
+
return fd
|
|
1306
|
+
|
|
1307
|
+
def _link_unnamed_at(self, fd: int, parent: int, name: str, *, where: str) -> None:
|
|
1308
|
+
"""Give the unnamed inode behind ``fd`` its first name, ``name`` under
|
|
1309
|
+
``parent``, through the process's own ``/proc/self/fd`` entry.
|
|
1310
|
+
|
|
1311
|
+
The inode is inspected immediately before: it must still have *no*
|
|
1312
|
+
name. A same-user process that can open this process's
|
|
1313
|
+
``/proc/<pid>/fd`` can name the inode the same way the controller is
|
|
1314
|
+
about to, so an inode found with any name is emptied through this
|
|
1315
|
+
descriptor and refused rather than published -- the same treatment a
|
|
1316
|
+
named temporary gets for a second name. ``linkat`` never replaces,
|
|
1317
|
+
so ``EEXIST`` on any existing entry (a symbolic link included) is
|
|
1318
|
+
surfaced as :class:`FileExistsError` for the caller to interpret.
|
|
1319
|
+
"""
|
|
1320
|
+
try:
|
|
1321
|
+
nlink = os.fstat(fd).st_nlink
|
|
1322
|
+
except OSError as exc:
|
|
1323
|
+
raise StateError(f"cannot write {where}: {exc}") from exc
|
|
1324
|
+
if nlink != 0:
|
|
1325
|
+
raise _empty_and_refuse(
|
|
1326
|
+
fd,
|
|
1327
|
+
where,
|
|
1328
|
+
f"being written through a temporary that acquired {nlink} directory "
|
|
1329
|
+
"entr(ies) before it was published; another process linked the "
|
|
1330
|
+
"controller's temporary file elsewhere",
|
|
1331
|
+
)
|
|
1332
|
+
try:
|
|
1333
|
+
os.link(_proc_fd_path(fd), name, dst_dir_fd=parent, follow_symlinks=True)
|
|
1334
|
+
except OSError as exc:
|
|
1335
|
+
if exc.errno == errno.EEXIST:
|
|
1336
|
+
raise FileExistsError(errno.EEXIST, f"{where} already exists") from exc
|
|
1337
|
+
raise _create_refusal(exc, where) from exc
|
|
1338
|
+
|
|
1339
|
+
def _name_unnamed_tmp_at(self, fd: int, parent: int, *, where: str) -> str:
|
|
1340
|
+
"""Give the unnamed inode behind ``fd`` a temporary name so that a
|
|
1341
|
+
rename can publish it, and return that name.
|
|
1342
|
+
|
|
1343
|
+
``linkat`` cannot replace an existing entry, so a whole-file
|
|
1344
|
+
replacement needs a named step after all: link, then rename. The
|
|
1345
|
+
name exists for exactly those two syscalls, with the bytes already
|
|
1346
|
+
complete, and the inode is re-inspected between them: it must have
|
|
1347
|
+
exactly the one name the controller just gave it. One that gained
|
|
1348
|
+
another is emptied through this descriptor, unlinked and refused.
|
|
1349
|
+
One that lost it -- the name removed from under the controller in
|
|
1350
|
+
that window -- is the same event the named path meets as the
|
|
1351
|
+
rename's ``ENOENT`` and is reported the same way, as a
|
|
1352
|
+
:class:`StateError`: nothing was planted, so there is nothing to
|
|
1353
|
+
refuse and nothing to move aside.
|
|
1354
|
+
|
|
1355
|
+
Once the name exists, every way out of this method other than
|
|
1356
|
+
returning it unlinks it: the re-inspection failing, the emptying
|
|
1357
|
+
failing, and the refusal itself. The caller never sees a name it
|
|
1358
|
+
does not receive, so a failure here leaves no ``.af-tmp-*`` behind.
|
|
1359
|
+
"""
|
|
1360
|
+
tmp = _tmp_name()
|
|
1361
|
+
try:
|
|
1362
|
+
self._link_unnamed_at(fd, parent, tmp, where=where)
|
|
1363
|
+
except FileExistsError as exc:
|
|
1364
|
+
# The random name collided; a fact about the directory, not the
|
|
1365
|
+
# target, so it is a plain failure of this write.
|
|
1366
|
+
raise StateError(f"cannot write {where}: temporary name {tmp} is taken") from exc
|
|
1367
|
+
try:
|
|
1368
|
+
nlink = os.fstat(fd).st_nlink
|
|
1369
|
+
if nlink == 0:
|
|
1370
|
+
raise StateError(
|
|
1371
|
+
f"cannot write {where}: the controller's temporary name {tmp} "
|
|
1372
|
+
"disappeared before it could be published"
|
|
1373
|
+
)
|
|
1374
|
+
if nlink > 1:
|
|
1375
|
+
raise _empty_and_refuse(
|
|
1376
|
+
fd,
|
|
1377
|
+
where,
|
|
1378
|
+
f"being written through a temporary that acquired {nlink - 1} extra "
|
|
1379
|
+
"directory entr(ies) before it was published; another process linked "
|
|
1380
|
+
"the controller's temporary file elsewhere",
|
|
1381
|
+
)
|
|
1382
|
+
except OSError as exc:
|
|
1383
|
+
_quiet_unlink(parent, tmp)
|
|
1384
|
+
raise StateError(f"cannot write {where}: {exc}") from exc
|
|
1385
|
+
except BaseException:
|
|
1386
|
+
_quiet_unlink(parent, tmp)
|
|
1387
|
+
raise
|
|
1388
|
+
return tmp
|
|
1389
|
+
|
|
1390
|
+
def _durable_tmp_at(self, parent: int, data: bytes, *, mode: int, where: str) -> str:
|
|
1391
|
+
"""Write ``data`` into a fresh, fsynced *named* temporary beside
|
|
1392
|
+
``where`` -- the path taken where :meth:`_unnamed_tmp_at` is not
|
|
1393
|
+
available.
|
|
1394
|
+
|
|
1395
|
+
Returns the temporary's name; the caller publishes it under the final
|
|
1396
|
+
name (or unlinks it). The name carries the pid and a random token, so
|
|
1397
|
+
two controllers and two attempts by one controller never collide.
|
|
1398
|
+
|
|
1399
|
+
The temporary is created ``O_EXCL`` -- it is this process's inode --
|
|
1400
|
+
but it is *named*, and a same-user process can hard-link that name
|
|
1401
|
+
elsewhere between the create and the write. So the descriptor is
|
|
1402
|
+
inspected again once the bytes are durable and before the name is
|
|
1403
|
+
handed to the caller: a temporary that has gained a second name is
|
|
1404
|
+
emptied through the descriptor it was written through, unlinked, and
|
|
1405
|
+
refused (see the module docstring for exactly what that closes and
|
|
1406
|
+
what it cannot).
|
|
1407
|
+
"""
|
|
1408
|
+
tmp = _tmp_name()
|
|
1409
|
+
fd = open_regular_at(
|
|
1410
|
+
parent, tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode=mode, where=where
|
|
1411
|
+
)
|
|
1412
|
+
try:
|
|
1413
|
+
with os.fdopen(fd, "wb") as fh:
|
|
1414
|
+
fh.write(data)
|
|
1415
|
+
fh.flush()
|
|
1416
|
+
os.fsync(fh.fileno())
|
|
1417
|
+
nlink = os.fstat(fh.fileno()).st_nlink
|
|
1418
|
+
if nlink > 1:
|
|
1419
|
+
# Our inode, emptied through our descriptor: the second
|
|
1420
|
+
# name keeps an empty file, and nothing is published.
|
|
1421
|
+
raise _empty_and_refuse(
|
|
1422
|
+
fh.fileno(),
|
|
1423
|
+
where,
|
|
1424
|
+
f"being written through a temporary that acquired {nlink - 1} extra "
|
|
1425
|
+
"directory entr(ies) while it was being written; another process "
|
|
1426
|
+
"linked the controller's temporary file elsewhere",
|
|
1427
|
+
)
|
|
1428
|
+
except OSError as exc:
|
|
1429
|
+
# The temporary is never published, so a failed write leaves
|
|
1430
|
+
# neither a partial final name nor a stray temporary behind.
|
|
1431
|
+
_quiet_unlink(parent, tmp)
|
|
1432
|
+
raise StateError(f"cannot write {where}: {exc}") from exc
|
|
1433
|
+
except BaseException:
|
|
1434
|
+
# A refusal, a failed emptying, or an interrupt: the name goes
|
|
1435
|
+
# the same way.
|
|
1436
|
+
_quiet_unlink(parent, tmp)
|
|
1437
|
+
raise
|
|
1438
|
+
return tmp
|
|
1439
|
+
|
|
1440
|
+
def _replace_at(self, parent: int, name: str, data: bytes, *, mode: int, where: str) -> None:
|
|
1441
|
+
fd = self._unnamed_tmp_at(parent, data, mode=mode, where=where)
|
|
1442
|
+
if fd is None:
|
|
1443
|
+
tmp = self._durable_tmp_at(parent, data, mode=mode, where=where)
|
|
1444
|
+
else:
|
|
1445
|
+
try:
|
|
1446
|
+
tmp = self._name_unnamed_tmp_at(fd, parent, where=where)
|
|
1447
|
+
finally:
|
|
1448
|
+
os.close(fd)
|
|
1449
|
+
try:
|
|
1450
|
+
os.replace(tmp, name, src_dir_fd=parent, dst_dir_fd=parent)
|
|
1451
|
+
except OSError as exc:
|
|
1452
|
+
_quiet_unlink(parent, tmp)
|
|
1453
|
+
if exc.errno == errno.EISDIR:
|
|
1454
|
+
raise _unsafe(where, "a directory, not a regular file") from exc
|
|
1455
|
+
raise StateError(f"cannot write {where}: {exc}") from exc
|
|
1456
|
+
self._fsync_published(parent, where)
|
|
1457
|
+
|
|
1458
|
+
def _publish_new_at(
|
|
1459
|
+
self, parent: int, name: str, data: bytes, *, mode: int, where: str
|
|
1460
|
+
) -> None:
|
|
1461
|
+
fd = self._unnamed_tmp_at(parent, data, mode=mode, where=where)
|
|
1462
|
+
if fd is not None:
|
|
1463
|
+
# The inode's first name is the final one: no temporary name at
|
|
1464
|
+
# any point, so nothing to unlink and nothing to observe.
|
|
1465
|
+
try:
|
|
1466
|
+
self._link_unnamed_at(fd, parent, name, where=where)
|
|
1467
|
+
finally:
|
|
1468
|
+
os.close(fd)
|
|
1469
|
+
self._fsync_published(parent, where)
|
|
1470
|
+
return
|
|
1471
|
+
tmp = self._durable_tmp_at(parent, data, mode=mode, where=where)
|
|
1472
|
+
try:
|
|
1473
|
+
os.link(tmp, name, src_dir_fd=parent, dst_dir_fd=parent, follow_symlinks=False)
|
|
1474
|
+
except OSError as exc:
|
|
1475
|
+
_quiet_unlink(parent, tmp)
|
|
1476
|
+
if exc.errno == errno.EEXIST:
|
|
1477
|
+
raise FileExistsError(errno.EEXIST, f"{where} already exists") from exc
|
|
1478
|
+
raise _create_refusal(exc, where) from exc
|
|
1479
|
+
_quiet_unlink(parent, tmp)
|
|
1480
|
+
self._fsync_published(parent, where)
|
|
1481
|
+
|
|
1482
|
+
@staticmethod
|
|
1483
|
+
def _fsync_published(parent: int, where: str) -> None:
|
|
1484
|
+
"""Make the directory entry just published at ``where`` durable.
|
|
1485
|
+
|
|
1486
|
+
The last step of every write that publishes a name. A filesystem
|
|
1487
|
+
that cannot fsync a directory is tolerated (:func:`_fsync_fd`); a
|
|
1488
|
+
fatal failure is reported through the controller's error type, as
|
|
1489
|
+
every other failure of the write is, rather than as a raw
|
|
1490
|
+
``OSError`` that the CLI's error boundary does not catch. The name
|
|
1491
|
+
is left in place: its bytes are whole and fsynced, and it is the
|
|
1492
|
+
entry's durability across a crash, not its contents, that could not
|
|
1493
|
+
be proved.
|
|
1494
|
+
"""
|
|
1495
|
+
try:
|
|
1496
|
+
_fsync_fd(parent)
|
|
1497
|
+
except OSError as exc:
|
|
1498
|
+
raise StateError(f"cannot durably publish {where}: {exc}") from exc
|
|
1499
|
+
|
|
1500
|
+
# -- walking ------------------------------------------------------------
|
|
1501
|
+
def walk(self, *, max_entries: int | None = None) -> Iterator[WalkEntry]:
|
|
1502
|
+
"""Yield every entry beneath the root, parents before children.
|
|
1503
|
+
|
|
1504
|
+
The walk descends only through descriptors it opened itself with
|
|
1505
|
+
``O_NOFOLLOW``, so it can neither be redirected out of the root by a
|
|
1506
|
+
symbolic link nor be made to follow one. Set ``entry.skip = True`` on
|
|
1507
|
+
a directory entry to stop the walk descending into it.
|
|
1508
|
+
|
|
1509
|
+
``max_entries`` bounds the *work*, not just the result: directory
|
|
1510
|
+
entries are counted as they are listed, before any of them is sorted,
|
|
1511
|
+
stat'ed or yielded, and the walk raises :class:`WalkBudgetExceeded`
|
|
1512
|
+
the moment the count passes the bound -- so one directory holding a
|
|
1513
|
+
million names costs a million ``readdir`` records and nothing more.
|
|
1514
|
+
The walk is iterative (an explicit stack, one descriptor per open
|
|
1515
|
+
level), so the tree's depth is bounded by the same budget rather than
|
|
1516
|
+
by the interpreter's recursion limit: a nest of ten thousand
|
|
1517
|
+
directories is ten thousand entries, not a ``RecursionError``.
|
|
1518
|
+
"""
|
|
1519
|
+
listed = 0
|
|
1520
|
+
# (directory descriptor, relpath prefix, names still to visit)
|
|
1521
|
+
stack: list[tuple[int, str, list[str]]] = []
|
|
1522
|
+
names = self._list_dir(self.fd, "", listed=0, max_entries=max_entries)
|
|
1523
|
+
listed += len(names)
|
|
1524
|
+
stack.append((self.fd, "", names))
|
|
1525
|
+
try:
|
|
1526
|
+
while stack:
|
|
1527
|
+
dir_fd, prefix, names = stack[-1]
|
|
1528
|
+
if not names:
|
|
1529
|
+
stack.pop()
|
|
1530
|
+
if dir_fd != self.fd:
|
|
1531
|
+
os.close(dir_fd)
|
|
1532
|
+
continue
|
|
1533
|
+
name = names.pop()
|
|
1534
|
+
relpath = f"{prefix}/{name}" if prefix else name
|
|
1535
|
+
try:
|
|
1536
|
+
st = os.lstat(name, dir_fd=dir_fd)
|
|
1537
|
+
except FileNotFoundError:
|
|
1538
|
+
# Vanished between the listing and the stat. A file that
|
|
1539
|
+
# is not there when we look is not part of the snapshot;
|
|
1540
|
+
# it cannot be, and pretending otherwise would be a lie
|
|
1541
|
+
# about what was read.
|
|
1542
|
+
continue
|
|
1543
|
+
except OSError as exc:
|
|
1544
|
+
if _denied(exc):
|
|
1545
|
+
raise UnreadableEntryError(
|
|
1546
|
+
relpath, f"cannot inspect {self._path / relpath}: {_denied_text(exc)}"
|
|
1547
|
+
) from exc
|
|
1548
|
+
raise StateError(f"cannot inspect {self._path / relpath}: {exc}") from exc
|
|
1549
|
+
entry = WalkEntry(relpath=relpath, name=name, dir_fd=dir_fd, st=st)
|
|
1550
|
+
yield entry
|
|
1551
|
+
if entry.skip or not entry.is_dir:
|
|
1552
|
+
continue
|
|
1553
|
+
try:
|
|
1554
|
+
child = os.open(name, _DIR_OPEN_FLAGS, dir_fd=dir_fd)
|
|
1555
|
+
except FileNotFoundError:
|
|
1556
|
+
continue
|
|
1557
|
+
except OSError as exc:
|
|
1558
|
+
if exc.errno in (errno.ELOOP, errno.EMLINK, errno.ENOTDIR):
|
|
1559
|
+
# lstat said directory, the O_NOFOLLOW open disagrees:
|
|
1560
|
+
# the entry was swapped underneath us.
|
|
1561
|
+
raise _unsafe(
|
|
1562
|
+
str(self._path / relpath), "not the directory it just was"
|
|
1563
|
+
) from exc
|
|
1564
|
+
if _denied(exc):
|
|
1565
|
+
raise UnreadableEntryError(
|
|
1566
|
+
relpath, f"cannot open {self._path / relpath}: {_denied_text(exc)}"
|
|
1567
|
+
) from exc
|
|
1568
|
+
raise StateError(f"cannot open {self._path / relpath}: {exc}") from exc
|
|
1569
|
+
try:
|
|
1570
|
+
children = self._list_dir(
|
|
1571
|
+
child, relpath, listed=listed, max_entries=max_entries
|
|
1572
|
+
)
|
|
1573
|
+
except BaseException:
|
|
1574
|
+
os.close(child)
|
|
1575
|
+
raise
|
|
1576
|
+
listed += len(children)
|
|
1577
|
+
stack.append((child, relpath, children))
|
|
1578
|
+
finally:
|
|
1579
|
+
for dir_fd, _, _ in stack:
|
|
1580
|
+
if dir_fd != self.fd:
|
|
1581
|
+
os.close(dir_fd)
|
|
1582
|
+
|
|
1583
|
+
def _list_dir(
|
|
1584
|
+
self, dir_fd: int, prefix: str, *, listed: int, max_entries: int | None
|
|
1585
|
+
) -> list[str]:
|
|
1586
|
+
"""The names in ``dir_fd``, sorted for iteration in reverse (``pop``).
|
|
1587
|
+
|
|
1588
|
+
``listed`` is how many entries the walk has already read elsewhere;
|
|
1589
|
+
with a ``max_entries`` the listing raises as soon as the total would
|
|
1590
|
+
pass it, without materialising the rest of the directory.
|
|
1591
|
+
"""
|
|
1592
|
+
names: list[str] = []
|
|
1593
|
+
try:
|
|
1594
|
+
with os.scandir(dir_fd) as it:
|
|
1595
|
+
for entry in it:
|
|
1596
|
+
if max_entries is not None and listed + len(names) >= max_entries:
|
|
1597
|
+
raise WalkBudgetExceeded(max_entries, prefix or ".")
|
|
1598
|
+
names.append(entry.name)
|
|
1599
|
+
except OSError as exc:
|
|
1600
|
+
where = self._path / prefix if prefix else self._path
|
|
1601
|
+
if _denied(exc):
|
|
1602
|
+
raise UnreadableEntryError(
|
|
1603
|
+
prefix or ".", f"cannot list {where}: {_denied_text(exc)}"
|
|
1604
|
+
) from exc
|
|
1605
|
+
raise StateError(f"cannot list {where}: {exc}") from exc
|
|
1606
|
+
names.sort(reverse=True)
|
|
1607
|
+
return names
|
|
1608
|
+
|
|
1609
|
+
|
|
1610
|
+
def _read_within(fh: BinaryIO, relpath: str, limit: int | None) -> bytes:
|
|
1611
|
+
"""Read all of ``fh``, or with a ``limit`` at most one byte past it.
|
|
1612
|
+
|
|
1613
|
+
A file the caller has decided is too large to hold is never
|
|
1614
|
+
materialised, whatever ``st_size`` claimed before it was opened: the
|
|
1615
|
+
read asks for ``limit + 1`` bytes and raises :class:`ReadLimitExceeded`
|
|
1616
|
+
when it gets them all.
|
|
1617
|
+
"""
|
|
1618
|
+
if limit is None:
|
|
1619
|
+
return fh.read()
|
|
1620
|
+
data = fh.read(limit + 1)
|
|
1621
|
+
if len(data) > limit:
|
|
1622
|
+
raise ReadLimitExceeded(relpath, limit)
|
|
1623
|
+
return data
|
|
1624
|
+
|
|
1625
|
+
|
|
1626
|
+
def _kind_at(dir_fd: int, name: str, fallback: str) -> str:
|
|
1627
|
+
"""Name what ``name`` actually is, for a refusal message.
|
|
1628
|
+
|
|
1629
|
+
Linux reports ``O_DIRECTORY | O_NOFOLLOW`` on a symbolic link as
|
|
1630
|
+
``ENOTDIR``, which would otherwise be reported as the unhelpful "not a
|
|
1631
|
+
directory". This is used only to word an error that has already been
|
|
1632
|
+
decided; it never influences whether the operation is refused.
|
|
1633
|
+
"""
|
|
1634
|
+
try:
|
|
1635
|
+
st = os.lstat(name, dir_fd=dir_fd)
|
|
1636
|
+
except OSError: # pragma: no cover - best effort wording
|
|
1637
|
+
return fallback
|
|
1638
|
+
kind = entry_kind(st.st_mode)
|
|
1639
|
+
if kind is None:
|
|
1640
|
+
return "a regular file, not a directory"
|
|
1641
|
+
if kind == "directory": # pragma: no cover - then the open would have worked
|
|
1642
|
+
return fallback
|
|
1643
|
+
return f"a {kind}, not a directory"
|
|
1644
|
+
|
|
1645
|
+
|
|
1646
|
+
def _supports_dir_fd() -> set[str]:
|
|
1647
|
+
return {fn.__name__ for fn in os.supports_dir_fd}
|
|
1648
|
+
|
|
1649
|
+
|
|
1650
|
+
def _mkdir_tree(p: Path) -> None:
|
|
1651
|
+
"""Create ``p`` and its parents by pathname (root creation only).
|
|
1652
|
+
|
|
1653
|
+
Only used to bring the *root* into existence; everything beneath a root
|
|
1654
|
+
is created through :meth:`SafeRoot.ensure_dir`, which is
|
|
1655
|
+
descriptor-relative.
|
|
1656
|
+
"""
|
|
1657
|
+
missing: list[Path] = []
|
|
1658
|
+
current = p
|
|
1659
|
+
while not current.exists():
|
|
1660
|
+
missing.append(current)
|
|
1661
|
+
parent = current.parent
|
|
1662
|
+
if parent == current:
|
|
1663
|
+
break
|
|
1664
|
+
current = parent
|
|
1665
|
+
try:
|
|
1666
|
+
os.makedirs(p, 0o700, exist_ok=True)
|
|
1667
|
+
for created in missing:
|
|
1668
|
+
_fsync_path(created)
|
|
1669
|
+
_fsync_path(created.parent)
|
|
1670
|
+
except OSError as exc:
|
|
1671
|
+
raise StateError(f"cannot create or durably publish controller root {p}: {exc}") from exc
|
|
1672
|
+
|
|
1673
|
+
|
|
1674
|
+
def _fsync_path(path: Path) -> None:
|
|
1675
|
+
fd = os.open(path, _DIR_OPEN_FLAGS & ~_O_NOFOLLOW)
|
|
1676
|
+
try:
|
|
1677
|
+
_fsync_fd(fd)
|
|
1678
|
+
finally:
|
|
1679
|
+
os.close(fd)
|
|
1680
|
+
|
|
1681
|
+
|
|
1682
|
+
def _fsync_fd(fd: int) -> None:
|
|
1683
|
+
try:
|
|
1684
|
+
os.fsync(fd)
|
|
1685
|
+
except OSError as exc:
|
|
1686
|
+
# Some filesystems reject fsync on directories. That is a weaker
|
|
1687
|
+
# durability guarantee, not a reason to fail a write. I/O, space and
|
|
1688
|
+
# permission failures are different: reporting success after one of
|
|
1689
|
+
# those would lose a checkpoint while the caller believes it durable.
|
|
1690
|
+
unsupported = {
|
|
1691
|
+
errno.EINVAL,
|
|
1692
|
+
errno.ENOSYS,
|
|
1693
|
+
getattr(errno, "ENOTSUP", errno.EINVAL),
|
|
1694
|
+
getattr(errno, "EOPNOTSUPP", errno.EINVAL),
|
|
1695
|
+
}
|
|
1696
|
+
if exc.errno in unsupported:
|
|
1697
|
+
return
|
|
1698
|
+
raise
|
|
1699
|
+
|
|
1700
|
+
|
|
1701
|
+
def _tmp_name() -> str:
|
|
1702
|
+
"""A temporary name that two controllers, and two attempts by one, never share."""
|
|
1703
|
+
return f".af-tmp-{os.getpid():x}-{secrets.token_hex(8)}"
|
|
1704
|
+
|
|
1705
|
+
|
|
1706
|
+
def _proc_fd_path(fd: int) -> str:
|
|
1707
|
+
"""The procfs name of this process's descriptor ``fd``: the only pathname
|
|
1708
|
+
that can denote an unnamed inode, and one only this process (and whatever
|
|
1709
|
+
may open its ``/proc/<pid>/fd``) can resolve."""
|
|
1710
|
+
return f"/proc/self/fd/{fd}"
|
|
1711
|
+
|
|
1712
|
+
|
|
1713
|
+
def _quiet_unlink(dir_fd: int, name: str) -> None:
|
|
1714
|
+
try:
|
|
1715
|
+
os.unlink(name, dir_fd=dir_fd)
|
|
1716
|
+
except OSError: # pragma: no cover - cleanup best effort
|
|
1717
|
+
pass
|