durablewrite 0.2.0__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {durablewrite-0.2.0 → durablewrite-1.0.0}/.gitignore +2 -0
- durablewrite-1.0.0/CHANGELOG.md +139 -0
- durablewrite-1.0.0/CONTRIBUTING.md +29 -0
- durablewrite-1.0.0/GUIDE.md +309 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/LICENSE +21 -21
- durablewrite-1.0.0/PKG-INFO +186 -0
- durablewrite-1.0.0/README.md +154 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/pyproject.toml +33 -7
- durablewrite-1.0.0/requirements-dev.txt +7 -0
- durablewrite-1.0.0/scripts/fetch_oracles.py +72 -0
- durablewrite-1.0.0/scripts/record_reference.py +57 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/__init__.py +1 -1
- durablewrite-1.0.0/src/durablewrite/_atomic.py +180 -0
- durablewrite-1.0.0/src/durablewrite/_check.py +98 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_cleanup.py +32 -14
- {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_cli.py +21 -9
- {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_errors.py +11 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_fs.py +44 -14
- {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_helpers.py +22 -0
- durablewrite-1.0.0/src/durablewrite/_options.py +116 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_replace.py +18 -3
- durablewrite-1.0.0/src/durablewrite/_win.py +54 -0
- durablewrite-1.0.0/src/durablewrite/_write.py +127 -0
- durablewrite-1.0.0/tests/_crashpoint.py +116 -0
- durablewrite-1.0.0/tests/_differential.py +86 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/_worker.py +6 -2
- durablewrite-1.0.0/tests/fixtures/atomicwrites-linux.json +1087 -0
- durablewrite-1.0.0/tests/fixtures/atomicwrites-windows.json +1087 -0
- durablewrite-1.0.0/tests/oracle/MANIFEST.json +187 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_concurrency.py +3 -2
- durablewrite-1.0.0/tests/test_contract_api.py +170 -0
- durablewrite-1.0.0/tests/test_contract_blind.py +170 -0
- durablewrite-1.0.0/tests/test_contract_blind_cleanup.py +103 -0
- durablewrite-1.0.0/tests/test_contract_blind_cli.py +80 -0
- durablewrite-1.0.0/tests/test_contract_cli.py +43 -0
- durablewrite-1.0.0/tests/test_contract_descriptors.py +126 -0
- durablewrite-1.0.0/tests/test_contract_edges.py +197 -0
- durablewrite-1.0.0/tests/test_contract_hostile.py +101 -0
- durablewrite-1.0.0/tests/test_contract_paths.py +184 -0
- durablewrite-1.0.0/tests/test_contract_platform.py +110 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_crash.py +26 -2
- durablewrite-1.0.0/tests/test_crash_points.py +117 -0
- durablewrite-1.0.0/tests/test_docs.py +90 -0
- durablewrite-1.0.0/tests/test_e2e_upgrade.py +79 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_e2e_wheel.py +3 -0
- durablewrite-1.0.0/tests/test_fuzz.py +173 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_hardening.py +18 -11
- durablewrite-1.0.0/tests/test_linux_trace.py +158 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_metadata.py +7 -1
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_options.py +2 -2
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_overwrite.py +6 -2
- durablewrite-1.0.0/tests/test_packaging.py +41 -0
- durablewrite-1.0.0/tests/test_reference.py +107 -0
- durablewrite-1.0.0/tests/test_sources.py +41 -0
- durablewrite-1.0.0/tests/trace/dwtrace.c +180 -0
- durablewrite-0.2.0/.gitattributes +0 -1
- durablewrite-0.2.0/.github/FUNDING.yml +0 -1
- durablewrite-0.2.0/CHANGELOG.md +0 -66
- durablewrite-0.2.0/CONTRIBUTING.md +0 -24
- durablewrite-0.2.0/GUIDE.md +0 -207
- durablewrite-0.2.0/IMPROVEMENTS-round.md +0 -93
- durablewrite-0.2.0/PKG-INFO +0 -171
- durablewrite-0.2.0/README.md +0 -144
- durablewrite-0.2.0/requirements-dev.txt +0 -3
- durablewrite-0.2.0/scripts/check.ps1 +0 -11
- durablewrite-0.2.0/scripts/release.ps1 +0 -28
- durablewrite-0.2.0/src/durablewrite/_atomic.py +0 -141
- durablewrite-0.2.0/src/durablewrite/_options.py +0 -79
- {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/__main__.py +0 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/py.typed +0 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/_payload.py +0 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/conftest.py +0 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_atomic.py +0 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_cli.py +0 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_contract.py +0 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_json_cleanup.py +0 -0
- {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_sharing.py +0 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
|
|
5
|
+
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [1.0.0] - 2026-10-16
|
|
10
|
+
|
|
11
|
+
The public API is now stable: the twelve names in `durablewrite.__all__`, their
|
|
12
|
+
parameters, defaults and error types, the temp file naming scheme that
|
|
13
|
+
`cleanup_temp_files` relies on, and the command line options and exit statuses
|
|
14
|
+
follow Semantic Versioning from here on.
|
|
15
|
+
|
|
16
|
+
### Changed
|
|
17
|
+
|
|
18
|
+
- `fsync="full"` (the default) on Windows renames with `MoveFileExW` and
|
|
19
|
+
`MOVEFILE_WRITE_THROUGH`, so the rename itself is written to disk before the
|
|
20
|
+
call returns; `"file"` and `"none"` keep plain `os.replace`.
|
|
21
|
+
- On macOS every flush uses `F_FULLFSYNC`, falling back to `fsync` where the
|
|
22
|
+
filesystem refuses it.
|
|
23
|
+
- All arguments are checked when a writer is called, before any file is
|
|
24
|
+
touched, instead of when the `with` block is entered.
|
|
25
|
+
- Error messages no longer repeat the refused value; the command line quotes at
|
|
26
|
+
most 40 characters of the values it checks and cuts every error or warning
|
|
27
|
+
message after its prefix to 120 characters, keeping its start and its end.
|
|
28
|
+
- On/off options (`overwrite`, `durable`, `fsync_dir`, `preserve_mode`,
|
|
29
|
+
`preserve_owner`, `follow_symlinks`) accept only `True` or `False`, and a `mode`
|
|
30
|
+
or `fsync` that is not a string raises `TypeError`: before, any value was taken
|
|
31
|
+
by truthiness, so `overwrite="no"` overwrote the file.
|
|
32
|
+
- `atomic_write` is typed per mode: `IO[str]` for `"w"`, `IO[bytes]` for `"wb"`
|
|
33
|
+
(`IO[Any]` only for a mode that is not a literal).
|
|
34
|
+
- `cleanup_temp_files` also removes read-only temp files on Windows, and
|
|
35
|
+
`older_than=0` removes every matching temp file whatever its time (on Windows
|
|
36
|
+
with Python 3.10 to 3.12 a file just written could look newer than the clock
|
|
37
|
+
and be skipped).
|
|
38
|
+
- Errors raised before the block runs name the path as it was given (for a
|
|
39
|
+
followed symlink, the file it points to); errors from
|
|
40
|
+
the rename name the target as an absolute path, since a relative target is
|
|
41
|
+
pinned to the working directory when the write starts.
|
|
42
|
+
- A `retry` that is not a `RetryPolicy` raises `TypeError` (0.2.0 raised
|
|
43
|
+
`AttributeError` and left a temp file).
|
|
44
|
+
- The `dev` extra is replaced by `requirements-dev.txt`.
|
|
45
|
+
|
|
46
|
+
### Fixed
|
|
47
|
+
|
|
48
|
+
- `cleanup_temp_files(older_than=float("nan"))` removed every temp file,
|
|
49
|
+
including those of running writers; a NaN or infinite age, a non-finite `now`
|
|
50
|
+
and non-number ages now raise.
|
|
51
|
+
- `RetryPolicy` accepted NaN and infinite delays, which made a blocked write
|
|
52
|
+
fail with an unrelated error and leave its temp file behind, non-integer or
|
|
53
|
+
`bool` attempts and a `sleep` that is not callable; these now raise
|
|
54
|
+
`TypeError` or `ValueError`.
|
|
55
|
+
- A FIFO, device or socket target was silently replaced by a regular file
|
|
56
|
+
(`durablewrite /dev/null` as root replaced the device; `--append` on a FIFO
|
|
57
|
+
hung); it now raises `OSError` with `EINVAL`.
|
|
58
|
+
- Permission bits were set on the temp file by name; on POSIX they now go
|
|
59
|
+
through the descriptor (`fchmod`), so a temp name swapped for a symlink cannot
|
|
60
|
+
redirect them.
|
|
61
|
+
- A text wrapper that failed to build closed the file descriptor twice and hid
|
|
62
|
+
its own error behind `EBADF`; codecs that are not text encodings (`hex`) are now
|
|
63
|
+
refused up front with `LookupError`.
|
|
64
|
+
- An interrupt between creating the temp file and wrapping it left the
|
|
65
|
+
descriptor open, so on Windows the temp file could not be removed.
|
|
66
|
+
- `--json` refused valid JSON with integers longer than 4300 digits.
|
|
67
|
+
- On Windows a name with `:` wrote to an alternate data stream and left an empty
|
|
68
|
+
file behind; it now raises `ValueError`.
|
|
69
|
+
- `RetryPolicy` accepted delays so long that `time.sleep` overflowed; delays are
|
|
70
|
+
now limited to one day.
|
|
71
|
+
- `bytes` paths, paths containing NUL, `permissions` outside `0..0o7777`,
|
|
72
|
+
`permissions=True`, unknown encodings and invalid `newline` values failed late
|
|
73
|
+
or with unrelated errors; they now raise `TypeError`, `ValueError` or `LookupError` up front.
|
|
74
|
+
|
|
75
|
+
### Added
|
|
76
|
+
|
|
77
|
+
- Ground truth in the test suite: kill-at-every-call tests on every platform, a
|
|
78
|
+
Linux system call tracer, 120 scenarios recorded from `atomicwrites` 1.4.1 on
|
|
79
|
+
Windows and Linux, and a manifest of the cited OS documentation.
|
|
80
|
+
- Doctests for every public name, seeded fuzz tests and linear-time checks.
|
|
81
|
+
|
|
82
|
+
## [0.2.0] - 2026-10-14
|
|
83
|
+
|
|
84
|
+
### Added
|
|
85
|
+
|
|
86
|
+
- `write_json_atomic(path, obj, ...)`: serialises before touching the disk and
|
|
87
|
+
writes UTF-8 JSON with a trailing newline and LF line endings. `NaN` and
|
|
88
|
+
infinities are refused unless `allow_nan=True`.
|
|
89
|
+
- `fsync="none" | "file" | "full"` on every writer, with the `FsyncLevel` type.
|
|
90
|
+
The `durable` and `fsync_dir` flags keep working.
|
|
91
|
+
- `overwrite=False`: atomic create-if-absent (`link` on POSIX, `MoveFileExW`
|
|
92
|
+
without the replace flag on Windows).
|
|
93
|
+
- `preserve_mode` (default on), `permissions` and `preserve_owner` options.
|
|
94
|
+
- `follow_symlinks` (default on): writing through a symlink updates the file it
|
|
95
|
+
points to and keeps the link.
|
|
96
|
+
- `RetryPolicy`, `DEFAULT_RETRY` and `NO_RETRY`: bounded backoff when a Windows
|
|
97
|
+
sharing or lock violation blocks the rename.
|
|
98
|
+
- `DurableWriteError` (an `OSError`), `SharingViolationError` (a
|
|
99
|
+
`PermissionError`) and `DirectorySyncError` (the file was replaced but its
|
|
100
|
+
directory could not be fsynced).
|
|
101
|
+
- `cleanup_temp_files(directory, older_than=..., target=...)` for temp files
|
|
102
|
+
left by killed writers.
|
|
103
|
+
- `durablewrite` command line tool (also `python -m durablewrite`): writes
|
|
104
|
+
standard input to a file atomically, with `--append`, `--json`, `--fsync`,
|
|
105
|
+
`--no-overwrite`, `--permissions`, `--no-follow-symlinks` and `--attempts`.
|
|
106
|
+
Exit status 1 on errors, 3 when the file was replaced but not fully synced.
|
|
107
|
+
- `GUIDE.md` with the write protocol, per-platform guarantees and sources.
|
|
108
|
+
|
|
109
|
+
### Changed
|
|
110
|
+
|
|
111
|
+
- Temp files are named `.<name>.<8 hex digits>.dwtmp` so they can be cleaned up
|
|
112
|
+
safely.
|
|
113
|
+
- Writing through a symlink no longer replaces the link with a regular file
|
|
114
|
+
(pass `follow_symlinks=False` for the old behaviour).
|
|
115
|
+
|
|
116
|
+
### Fixed
|
|
117
|
+
|
|
118
|
+
- New files were created with mode `0600` and replaced files lost their mode;
|
|
119
|
+
new files now honour the umask like `open()` and replaced files keep their
|
|
120
|
+
permission bits.
|
|
121
|
+
- Very long file names no longer overflow the name limit when the temp name is
|
|
122
|
+
built.
|
|
123
|
+
- Writing to an existing directory raises `IsADirectoryError` before any temp
|
|
124
|
+
file is created, and a path ending with a separator raises `ValueError`.
|
|
125
|
+
- When the final mode is restricted (explicit `permissions` or a replaced
|
|
126
|
+
private file) the temp file is `0600` while it is written.
|
|
127
|
+
- On Windows, removal of a temp file that is briefly locked is retried.
|
|
128
|
+
|
|
129
|
+
## [0.1.0] - 2026-10-11 (not published)
|
|
130
|
+
|
|
131
|
+
### Added
|
|
132
|
+
|
|
133
|
+
- `atomic_write(path, mode, *, encoding, newline, durable, fsync_dir)` context
|
|
134
|
+
manager: writes to a temp file in the target directory, flushes and fsyncs it,
|
|
135
|
+
then moves it onto the target with `os.replace` and fsyncs the directory.
|
|
136
|
+
- `write_text_atomic(path, text, encoding, *, newline, durable, fsync_dir)` and
|
|
137
|
+
`write_bytes_atomic(path, data, *, durable, fsync_dir)` one-shot helpers.
|
|
138
|
+
- Durable by default (file and directory fsync), both switchable off.
|
|
139
|
+
- Correct on Windows and POSIX; typed, zero runtime dependencies.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Thanks for taking the time. This project keeps a small, predictable shape.
|
|
4
|
+
|
|
5
|
+
## Setup
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
pip install -r requirements-dev.txt
|
|
9
|
+
python scripts/fetch_oracles.py # optional: reference library and cited pages
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Before you open a pull request
|
|
13
|
+
|
|
14
|
+
- Keep public API in `src/durablewrite/__init__.py`; internals live in `_`-prefixed modules.
|
|
15
|
+
- Add a test for the behaviour you change. Tests run against the real filesystem and
|
|
16
|
+
real child processes; only the retry `sleep` and, in a few contract tests, single
|
|
17
|
+
OS calls are replaced to observe what is asked of the operating system.
|
|
18
|
+
- Run the gate: `ruff format --check .`, `ruff check .`, `mypy --strict src tests` and
|
|
19
|
+
`pytest` (the repository has `scripts/check.ps1` for this). The multi-process and
|
|
20
|
+
packaging tests are marked `slow`; `pytest -m "not slow"` skips them while
|
|
21
|
+
iterating, but the full suite must pass before a commit. Run it on Windows and on
|
|
22
|
+
Linux; the system call tests need Linux and `gcc`.
|
|
23
|
+
- The crash and fuzz tests are seeded: set `DURABLEWRITE_CRASH_SEED`,
|
|
24
|
+
`DURABLEWRITE_CRASH_ITERATIONS` or `DURABLEWRITE_FUZZ_SEED` to explore further.
|
|
25
|
+
- Material fetched by `scripts/fetch_oracles.py` stays out of the repository; only
|
|
26
|
+
`tests/oracle/MANIFEST.json` is tracked.
|
|
27
|
+
- One logical change per commit, imperative subject line.
|
|
28
|
+
|
|
29
|
+
Open an issue first if the change is large or shifts the public API.
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
# durablewrite guide
|
|
2
|
+
|
|
3
|
+
This guide explains what `durablewrite` does on every write, what it guarantees
|
|
4
|
+
on each platform, and where those guarantees stop. Each claim names its source
|
|
5
|
+
or the test in this repository that exercises it.
|
|
6
|
+
|
|
7
|
+
## The write protocol
|
|
8
|
+
|
|
9
|
+
For `atomic_write(path)` and every helper built on it:
|
|
10
|
+
|
|
11
|
+
1. **Check the arguments** when the writer is called, before any file is
|
|
12
|
+
touched (see "Errors" below).
|
|
13
|
+
2. **Resolve the target.** If `path` is a symlink and `follow_symlinks=True`
|
|
14
|
+
(the default), the file it points to becomes the target, so the link keeps
|
|
15
|
+
working. With `follow_symlinks=False` the link itself is replaced by a
|
|
16
|
+
regular file. An existing directory is rejected with `IsADirectoryError`, a
|
|
17
|
+
FIFO, device or socket with `OSError` (`EINVAL`) so it is never swapped for a
|
|
18
|
+
regular file, and
|
|
19
|
+
with `overwrite=False` an existing name is rejected with `FileExistsError`,
|
|
20
|
+
both before the block runs.
|
|
21
|
+
3. **Create a temp file in the target's own directory**, named
|
|
22
|
+
`.<name>.<8 hex digits>.dwtmp`, with `O_CREAT | O_EXCL`. A new file is created
|
|
23
|
+
with mode `0o666`, so the process umask applies exactly as it does for
|
|
24
|
+
`open()`. When the final mode comes from `permissions` or from the replaced
|
|
25
|
+
file, the temp file starts as `0o600` and is widened only after the data is
|
|
26
|
+
written, so restricted content is never readable by others in between. Names
|
|
27
|
+
longer than 48 characters keep a readable head plus a hash of the full name,
|
|
28
|
+
so the temp name stays within the 255-byte limit. The same directory means
|
|
29
|
+
the same filesystem, which the rename needs: Python documents that
|
|
30
|
+
`os.replace` may fail across filesystems [1].
|
|
31
|
+
4. **Write** your data into it.
|
|
32
|
+
5. **Set metadata.** With `preserve_owner=True` (POSIX) the uid and gid of the
|
|
33
|
+
old file are copied first, then the permission bits: either `permissions` or,
|
|
34
|
+
with `preserve_mode=True` (the default), the bits of the old file. On POSIX
|
|
35
|
+
both go through the open descriptor (`fchown`, `fchmod`), so a temp name
|
|
36
|
+
swapped for a symlink by someone else cannot redirect them.
|
|
37
|
+
6. **Flush the temp file to disk** (`fsync="file"` or `"full"`): `fsync` on
|
|
38
|
+
Linux, `F_FULLFSYNC` on macOS, `FlushFileBuffers` on Windows. This must come
|
|
39
|
+
before the rename, or the new name can reach the disk before the data it
|
|
40
|
+
points to, which is how ext4 left zero-length files after a crash until it
|
|
41
|
+
added a heuristic for that pattern [3][4].
|
|
42
|
+
7. **Rename** the temp file onto the target. With `overwrite=False` a
|
|
43
|
+
create-only rename is used instead (see below).
|
|
44
|
+
8. **Make the rename durable** (`fsync="full"`). On POSIX the parent directory is
|
|
45
|
+
fsynced: the Linux `fsync(2)` page says a file fsync does not by itself make
|
|
46
|
+
the directory entry durable, and asks for an fsync of the directory [2][3].
|
|
47
|
+
On Windows the rename of step 7 is made with `MOVEFILE_WRITE_THROUGH`, which
|
|
48
|
+
Microsoft documents as not returning until the move is on the disk [6].
|
|
49
|
+
|
|
50
|
+
If anything raises before step 7 completes, the temp file is removed and the
|
|
51
|
+
target is untouched. That includes exceptions inside your `with` block and
|
|
52
|
+
`KeyboardInterrupt`.
|
|
53
|
+
|
|
54
|
+
## What is guaranteed, per platform
|
|
55
|
+
|
|
56
|
+
| Promise | Linux / macOS (local fs) | Windows (local NTFS) |
|
|
57
|
+
| --- | --- | --- |
|
|
58
|
+
| Readers see the old or the complete new file, never a mix | Yes, `rename(2)` replaces the name atomically [5][12] | Yes on NTFS in our tests; Microsoft does not call the rename atomic [6] |
|
|
59
|
+
| A killed writer leaves the old or the new file, never a partial one | Yes (tested at every call) | Yes (tested at every call) |
|
|
60
|
+
| New content survives power loss once the call returns (`"full"`) | Linux: data and directory fsynced. macOS: `F_FULLFSYNC` [8] | Data flushed [7]; rename written through [6] |
|
|
61
|
+
| Permission bits kept on replace | Yes (`preserve_mode`) | Not applicable; see below |
|
|
62
|
+
| Owner kept on replace | Opt-in (`preserve_owner`), needs privilege to change uid | No |
|
|
63
|
+
| Replace while another process has the target open | Yes; the reader keeps the old inode | Retried, then `SharingViolationError` |
|
|
64
|
+
| A reader can always open the target | Yes | No: an `open()` racing a rename can fail for a moment with `PermissionError` or, rarely, `FileNotFoundError` (never torn); readers that poll should retry both |
|
|
65
|
+
| Replace a read-only target | Yes, if the directory is writable | No, `PermissionError` (not retried) |
|
|
66
|
+
|
|
67
|
+
### Process crash versus power loss
|
|
68
|
+
|
|
69
|
+
A process crash is tested directly. `tests/test_crash_points.py` runs a writer
|
|
70
|
+
that kills itself with no cleanup right before and right after each file system
|
|
71
|
+
call it makes, for every call, on Windows and POSIX. `tests/test_linux_trace.py`
|
|
72
|
+
does the same on Linux one level lower: an `LD_PRELOAD` library
|
|
73
|
+
(`tests/trace/dwtrace.c`) logs the real libc calls (`open`, `write`, `fsync`,
|
|
74
|
+
`rename`, `link`, `unlink`) and sends `SIGKILL` at each boundary. In every case
|
|
75
|
+
the target holds the old content until the rename has returned and the new
|
|
76
|
+
content from then on, and any leftover temp file is one that
|
|
77
|
+
`cleanup_temp_files` removes. `tests/test_crash.py` also kills writers at random
|
|
78
|
+
moments mid-write.
|
|
79
|
+
|
|
80
|
+
No portable test can cut the power, so durability against power loss rests on
|
|
81
|
+
the documented calls above and on the filesystem honouring them [3].
|
|
82
|
+
|
|
83
|
+
### Linux
|
|
84
|
+
|
|
85
|
+
`fsync="full"` follows the sequence recommended in "Ensuring data reaches disk"
|
|
86
|
+
[3]: fsync the file, rename, fsync the directory. The trace test records exactly
|
|
87
|
+
that order. Do not rely on ext4's `auto_da_alloc` heuristic for "replace via
|
|
88
|
+
rename" [4]; it is an ext4 mount option that can be turned off, not a promise of
|
|
89
|
+
the rename itself. If the directory fsync raises (some network and FUSE filesystems refuse it,
|
|
90
|
+
and it fails when the directory is writable but not readable), the new content is
|
|
91
|
+
already in place and `DirectorySyncError` is raised so you know the rename may
|
|
92
|
+
not be durable (`tests/test_hardening.py`).
|
|
93
|
+
|
|
94
|
+
### macOS
|
|
95
|
+
|
|
96
|
+
Apple documents that `fsync` hands the data to the drive without asking the
|
|
97
|
+
drive to empty its own write cache, and offers the `F_FULLFSYNC` fcntl for that
|
|
98
|
+
[8]. `durablewrite` issues `F_FULLFSYNC` for the file and for the directory, and
|
|
99
|
+
falls back to `fsync` on a filesystem that refuses it. This is the same choice
|
|
100
|
+
`atomicwrites` made. macOS was not part of this release's test runs; the code
|
|
101
|
+
path is pinned by `tests/test_contract_platform.py`.
|
|
102
|
+
|
|
103
|
+
### Windows
|
|
104
|
+
|
|
105
|
+
- `os.fsync` calls the C runtime's `_commit` [1][9], which forces the file to
|
|
106
|
+
disk; on Windows 11 a hook on the C runtime showed `_commit` calling
|
|
107
|
+
`FlushFileBuffers` [7] (observed, not documented on the `_commit` page).
|
|
108
|
+
- With `fsync="full"`, the rename is `MoveFileExW` with
|
|
109
|
+
`MOVEFILE_WRITE_THROUGH` (plus `MOVEFILE_REPLACE_EXISTING` unless
|
|
110
|
+
`overwrite=False`) [6], as `atomicwrites` does. With `"file"` or `"none"` it is
|
|
111
|
+
plain `os.replace`, which is `MoveFileExW` without write-through.
|
|
112
|
+
- There is no directory fsync through the APIs Python exposes; write-through
|
|
113
|
+
takes its place.
|
|
114
|
+
- Microsoft does not document the rename as atomic. On a local NTFS volume it is
|
|
115
|
+
a single metadata operation and our tests never observed a torn target. A
|
|
116
|
+
reader opening the name while it is being replaced can, rarely, get
|
|
117
|
+
`FileNotFoundError` (about 1 open in 1,000 to 2,000 with four writers
|
|
118
|
+
hammering one file); 0.2.0 behaved the same. On FAT volumes and SMB shares the behaviour depends on the filesystem
|
|
119
|
+
or server and is not guaranteed.
|
|
120
|
+
- The rename fails with `ERROR_ACCESS_DENIED`, `ERROR_SHARING_VIOLATION` or
|
|
121
|
+
`ERROR_LOCK_VIOLATION` while any process holds the target open without
|
|
122
|
+
`FILE_SHARE_DELETE` [10]. Python's `open()` does not pass that flag, and
|
|
123
|
+
antivirus scanners and the search indexer briefly open new files. These errors
|
|
124
|
+
are retried with bounded backoff (`RetryPolicy`, about 1.1 seconds by default).
|
|
125
|
+
A read-only target is never retried. Several processes replacing one file
|
|
126
|
+
while others read it can keep it locked for longer than that on a slow or busy
|
|
127
|
+
machine, and so can many writers of one file on their own (an adversarial run
|
|
128
|
+
of 16 threads saw a few blocked renames in 3,200); give such writers a larger
|
|
129
|
+
`RetryPolicy`. When the budget runs out,
|
|
130
|
+
`SharingViolationError` is raised and the target is untouched
|
|
131
|
+
(`tests/test_sharing.py`). A directory ACL that denies the rename produces the
|
|
132
|
+
same error code and ends the same way.
|
|
133
|
+
- The temp file can itself be held open (an antivirus scan of the fresh file).
|
|
134
|
+
On every error path its removal is retried under the same `RetryPolicy`; if it
|
|
135
|
+
is still locked after that, it is left for `cleanup_temp_files`, which also
|
|
136
|
+
clears a read-only attribute before removing.
|
|
137
|
+
- Permission bits do not exist on Windows. `permissions` only toggles the
|
|
138
|
+
read-only attribute and `preserve_mode` does nothing. The new file inherits the
|
|
139
|
+
ACL of its directory; an explicit ACL on the old file is not copied.
|
|
140
|
+
|
|
141
|
+
### Why not `ReplaceFileW`
|
|
142
|
+
|
|
143
|
+
`ReplaceFileW` [11] keeps the replaced file's ACL, attributes and creation time,
|
|
144
|
+
but it is built around an existing file to replace (in our tests on Windows 11
|
|
145
|
+
it failed when the target was missing) and it can fail half way: Microsoft lists a
|
|
146
|
+
result, `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2`, in which the replaced file is left
|
|
147
|
+
under another name. A primitive whose failure mode removes the target conflicts
|
|
148
|
+
with the core promise, so `durablewrite` uses `MoveFileExW`.
|
|
149
|
+
|
|
150
|
+
## Create only if absent
|
|
151
|
+
|
|
152
|
+
`overwrite=False` publishes the complete file only if nothing exists at the
|
|
153
|
+
target, and raises `FileExistsError` otherwise, without a check-then-act race:
|
|
154
|
+
|
|
155
|
+
- POSIX: `link(temp, target)` fails with `EEXIST` if the name is taken [13], then
|
|
156
|
+
the temp name is unlinked. The filesystem must support hard links.
|
|
157
|
+
- Windows: `MoveFileExW` without `MOVEFILE_REPLACE_EXISTING` fails with error
|
|
158
|
+
183 (`FileExistsError`), observed on NTFS (`tests/test_contract_platform.py`).
|
|
159
|
+
|
|
160
|
+
An existing name is refused before the block runs; a name that appears while
|
|
161
|
+
the block runs is refused by the rename. `tests/test_overwrite.py` races six
|
|
162
|
+
processes for one name and checks that exactly one wins.
|
|
163
|
+
|
|
164
|
+
## Symlinks
|
|
165
|
+
|
|
166
|
+
The default (`follow_symlinks=True`) writes to the file the link points to and
|
|
167
|
+
creates the temp file beside that file, so the rename never crosses a
|
|
168
|
+
filesystem even when the link lives elsewhere. A dangling link creates its
|
|
169
|
+
target. `follow_symlinks=False` replaces the link with a regular file, which is
|
|
170
|
+
what plain `os.replace` (and `atomicwrites`) does; on POSIX this also works when
|
|
171
|
+
the link points to a directory. The replaced link counts as a new file, so it
|
|
172
|
+
does not inherit the mode of whatever it pointed to.
|
|
173
|
+
|
|
174
|
+
## Side effects of replacing a file
|
|
175
|
+
|
|
176
|
+
Replacing a file creates a new file, by design. As a consequence:
|
|
177
|
+
|
|
178
|
+
- Other hard links to the old file keep the old content.
|
|
179
|
+
- The inode (POSIX) or file ID (Windows) changes. Programs that follow a file by
|
|
180
|
+
handle, such as `tail -f` without `-F`, keep reading the old file.
|
|
181
|
+
- Extended attributes, POSIX ACLs and Windows alternate data streams of the old
|
|
182
|
+
file are not copied.
|
|
183
|
+
|
|
184
|
+
## Leftover temp files
|
|
185
|
+
|
|
186
|
+
A writer that is killed between creating its temp file and renaming it cannot
|
|
187
|
+
clean up. `cleanup_temp_files(directory, older_than=3600)` removes only regular
|
|
188
|
+
files matching `.<name>.<8 hex digits>.dwtmp` older than the given age, by
|
|
189
|
+
modification time. A writer that keeps its temp file open for longer than that
|
|
190
|
+
age could lose it, so choose an age longer than your slowest write. With
|
|
191
|
+
`target=`, only temp files of that exact target name are considered. The age
|
|
192
|
+
must be a finite number of seconds, not negative: `NaN` once made every temp
|
|
193
|
+
file look stale, so it is refused.
|
|
194
|
+
|
|
195
|
+
## Errors
|
|
196
|
+
|
|
197
|
+
| Error | When |
|
|
198
|
+
| --- | --- |
|
|
199
|
+
| `TypeError` | a `bytes` path, an on/off option that is not `True` or `False`, or another argument of the wrong type |
|
|
200
|
+
| `ValueError` | a bad `mode`, `fsync`, `newline`, `permissions`, retry setting (delays from 0 to one day) or age; an empty path, a NUL in it, one ending with a separator, or on Windows a name with `:` (an alternate data stream) |
|
|
201
|
+
| `LookupError` | an unknown `encoding`, or a codec that is not a text encoding (`hex`) |
|
|
202
|
+
| `IsADirectoryError` | the target is a directory |
|
|
203
|
+
| `OSError` (`EINVAL`) | the target is a FIFO, device or socket |
|
|
204
|
+
| `FileExistsError` | `overwrite=False` and the name exists |
|
|
205
|
+
| `SharingViolationError` | Windows kept the rename blocked for the whole retry budget (a `PermissionError`) |
|
|
206
|
+
| `DirectorySyncError` | the file was replaced, but the directory fsync failed (POSIX, `"full"`) |
|
|
207
|
+
| other `OSError` | whatever the operating system reports: a missing directory, no space, no permission |
|
|
208
|
+
|
|
209
|
+
The first three are raised when the writer is called, before any file is
|
|
210
|
+
touched. Messages are short and never repeat the value they refuse.
|
|
211
|
+
`DurableWriteError` is the base of the two errors `durablewrite` raises itself
|
|
212
|
+
and is an `OSError`, so existing `except OSError` code keeps working.
|
|
213
|
+
|
|
214
|
+
## Command line
|
|
215
|
+
|
|
216
|
+
`durablewrite TARGET` copies standard input into TARGET with the protocol above,
|
|
217
|
+
like `sponge` from moreutils. TARGET is replaced only after standard input ends,
|
|
218
|
+
so `sort f | durablewrite f` is safe on every platform, and
|
|
219
|
+
`durablewrite --append f < f` works on POSIX. On Windows the shell (and the
|
|
220
|
+
console-script launcher) keep a redirected file open without
|
|
221
|
+
`FILE_SHARE_DELETE`, which blocks the rename, so the command detects that
|
|
222
|
+
standard input is TARGET and refuses at once; pipe the file instead.
|
|
223
|
+
|
|
224
|
+
Exit status: 0 on success; 1 on any error, including a usage error, with TARGET
|
|
225
|
+
left as it was; 3 when TARGET was replaced but its directory could not be
|
|
226
|
+
fsynced; 130 when interrupted. `--json` accepts only strict JSON (no `NaN` or
|
|
227
|
+
`Infinity`). Each `--append` run is atomic, but two concurrent runs on the same
|
|
228
|
+
file can lose one run's data, because each starts from the content it read.
|
|
229
|
+
|
|
230
|
+
## How this was checked
|
|
231
|
+
|
|
232
|
+
- **Documentation**: `tests/oracle/MANIFEST.json` lists every page cited below
|
|
233
|
+
with its URL, fetch date, size and hash; `scripts/fetch_oracles.py` downloads
|
|
234
|
+
them into a local folder that is never committed, and `tests/test_sources.py`
|
|
235
|
+
checks that each still names the calls and flags this guide relies on.
|
|
236
|
+
- **System calls**: `tests/test_linux_trace.py` (Linux, needs gcc) records the
|
|
237
|
+
real call order for each fsync level and kills the writer at every call.
|
|
238
|
+
- **Reference library**: `atomicwrites` 1.4.1 ran 120 scenarios (new file,
|
|
239
|
+
replace, create-only, directory, symlink, dangling link, failing block, text and
|
|
240
|
+
binary) on Windows and Linux; `tests/fixtures/atomicwrites-*.json` is its
|
|
241
|
+
recorded output, and `tests/test_reference.py` checks that durablewrite matches
|
|
242
|
+
it wherever a plain file is written, and differs only as this guide says
|
|
243
|
+
elsewhere. Under the tracer both libraries make the same fsync, rename,
|
|
244
|
+
directory fsync sequence.
|
|
245
|
+
|
|
246
|
+
## Upgrading from 0.2
|
|
247
|
+
|
|
248
|
+
Code that passes the documented argument types keeps working; the differences:
|
|
249
|
+
|
|
250
|
+
- `fsync="full"` now uses `MOVEFILE_WRITE_THROUGH` on Windows and `F_FULLFSYNC`
|
|
251
|
+
on macOS.
|
|
252
|
+
- Arguments are checked when the writer is called rather than when the `with`
|
|
253
|
+
block is entered, and more of them are checked: `bytes` paths, paths with a
|
|
254
|
+
NUL, `permissions` outside `0..0o7777`, unknown encodings, invalid `newline`
|
|
255
|
+
values, codecs that are not text encodings, a `retry` that is not a
|
|
256
|
+
`RetryPolicy`, and on Windows alternate data stream names.
|
|
257
|
+
- A FIFO, device or socket target raises `OSError` (`EINVAL`); 0.2.0 replaced it
|
|
258
|
+
with a regular file (as root, `durablewrite /dev/null` replaced the device).
|
|
259
|
+
- `RetryPolicy` delays longer than one day are refused.
|
|
260
|
+
- `--json` checks numbers without converting them, so integers longer than 4300
|
|
261
|
+
digits are accepted.
|
|
262
|
+
- On/off options (`overwrite`, `durable`, `fsync_dir`, `preserve_mode`,
|
|
263
|
+
`preserve_owner`, `follow_symlinks`) must be `True` or `False`: 0.2.0 took any
|
|
264
|
+
value by truthiness, so `overwrite="no"` overwrote. A `mode` or `fsync` that is
|
|
265
|
+
not a string, and `permissions=True`, raise `TypeError`.
|
|
266
|
+
- `RetryPolicy` refuses attempts that are not an `int` (a `bool` included), a
|
|
267
|
+
`sleep` that is not callable, and delays or multipliers that are NaN, infinite
|
|
268
|
+
or of the wrong type; `cleanup_temp_files` refuses a NaN or
|
|
269
|
+
infinite age and a non-finite `now`.
|
|
270
|
+
- `cleanup_temp_files` removes read-only temp files on Windows, and
|
|
271
|
+
`older_than=0` removes every matching temp file whatever its time.
|
|
272
|
+
- Errors raised before the block runs name the path as given (for a followed
|
|
273
|
+
symlink, the file it points to); errors from the rename name the absolute
|
|
274
|
+
target (a relative target is pinned when the write
|
|
275
|
+
starts).
|
|
276
|
+
- Error messages no longer repeat the refused value. The command line quotes at
|
|
277
|
+
most 40 characters of the values it checks itself, and every error or warning
|
|
278
|
+
message it prints after its `durablewrite: error:` prefix is cut to 120
|
|
279
|
+
characters, keeping its start and its end (where the file name is).
|
|
280
|
+
- `atomic_write` is typed per mode: `IO[str]` for `"w"`, `IO[bytes]` for `"wb"`.
|
|
281
|
+
- The `dev` extra is gone; contributors use `requirements-dev.txt`.
|
|
282
|
+
|
|
283
|
+
## Sources
|
|
284
|
+
|
|
285
|
+
Fetched on 2026-10-16.
|
|
286
|
+
|
|
287
|
+
1. Python documentation, `os.replace` and `os.fsync`:
|
|
288
|
+
https://docs.python.org/3/library/os.html
|
|
289
|
+
2. Linux `fsync(2)` manual page: https://man7.org/linux/man-pages/man2/fsync.2.html
|
|
290
|
+
3. Jeff Moyer, "Ensuring data reaches disk", LWN.net, 2011:
|
|
291
|
+
https://lwn.net/Articles/457667/
|
|
292
|
+
4. Linux kernel documentation, ext4 mount options (`auto_da_alloc`):
|
|
293
|
+
https://docs.kernel.org/admin-guide/ext4.html
|
|
294
|
+
5. POSIX.1-2024, `rename`:
|
|
295
|
+
https://pubs.opengroup.org/onlinepubs/9799919799/functions/rename.html
|
|
296
|
+
6. Microsoft, `MoveFileExW`:
|
|
297
|
+
https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-movefileexw
|
|
298
|
+
7. Microsoft, `FlushFileBuffers`:
|
|
299
|
+
https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-flushfilebuffers
|
|
300
|
+
8. Apple, `fsync(2)` manual page (`F_FULLFSYNC`):
|
|
301
|
+
https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fsync.2.html
|
|
302
|
+
9. Microsoft, `_commit`:
|
|
303
|
+
https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/commit
|
|
304
|
+
10. Microsoft, `CreateFileW` (`FILE_SHARE_DELETE`):
|
|
305
|
+
https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
|
|
306
|
+
11. Microsoft, `ReplaceFileW`:
|
|
307
|
+
https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-replacefilew
|
|
308
|
+
12. Linux `rename(2)` manual page: https://man7.org/linux/man-pages/man2/rename.2.html
|
|
309
|
+
13. Linux `link(2)` manual page: https://man7.org/linux/man-pages/man2/link.2.html
|
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2026 Javier Lianes
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Javier Lianes
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|