durablewrite 0.2.0__tar.gz → 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. {durablewrite-0.2.0 → durablewrite-1.0.0}/.gitignore +2 -0
  2. durablewrite-1.0.0/CHANGELOG.md +139 -0
  3. durablewrite-1.0.0/CONTRIBUTING.md +29 -0
  4. durablewrite-1.0.0/GUIDE.md +309 -0
  5. {durablewrite-0.2.0 → durablewrite-1.0.0}/LICENSE +21 -21
  6. durablewrite-1.0.0/PKG-INFO +186 -0
  7. durablewrite-1.0.0/README.md +154 -0
  8. {durablewrite-0.2.0 → durablewrite-1.0.0}/pyproject.toml +33 -7
  9. durablewrite-1.0.0/requirements-dev.txt +7 -0
  10. durablewrite-1.0.0/scripts/fetch_oracles.py +72 -0
  11. durablewrite-1.0.0/scripts/record_reference.py +57 -0
  12. {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/__init__.py +1 -1
  13. durablewrite-1.0.0/src/durablewrite/_atomic.py +180 -0
  14. durablewrite-1.0.0/src/durablewrite/_check.py +98 -0
  15. {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_cleanup.py +32 -14
  16. {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_cli.py +21 -9
  17. {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_errors.py +11 -0
  18. {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_fs.py +44 -14
  19. {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_helpers.py +22 -0
  20. durablewrite-1.0.0/src/durablewrite/_options.py +116 -0
  21. {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/_replace.py +18 -3
  22. durablewrite-1.0.0/src/durablewrite/_win.py +54 -0
  23. durablewrite-1.0.0/src/durablewrite/_write.py +127 -0
  24. durablewrite-1.0.0/tests/_crashpoint.py +116 -0
  25. durablewrite-1.0.0/tests/_differential.py +86 -0
  26. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/_worker.py +6 -2
  27. durablewrite-1.0.0/tests/fixtures/atomicwrites-linux.json +1087 -0
  28. durablewrite-1.0.0/tests/fixtures/atomicwrites-windows.json +1087 -0
  29. durablewrite-1.0.0/tests/oracle/MANIFEST.json +187 -0
  30. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_concurrency.py +3 -2
  31. durablewrite-1.0.0/tests/test_contract_api.py +170 -0
  32. durablewrite-1.0.0/tests/test_contract_blind.py +170 -0
  33. durablewrite-1.0.0/tests/test_contract_blind_cleanup.py +103 -0
  34. durablewrite-1.0.0/tests/test_contract_blind_cli.py +80 -0
  35. durablewrite-1.0.0/tests/test_contract_cli.py +43 -0
  36. durablewrite-1.0.0/tests/test_contract_descriptors.py +126 -0
  37. durablewrite-1.0.0/tests/test_contract_edges.py +197 -0
  38. durablewrite-1.0.0/tests/test_contract_hostile.py +101 -0
  39. durablewrite-1.0.0/tests/test_contract_paths.py +184 -0
  40. durablewrite-1.0.0/tests/test_contract_platform.py +110 -0
  41. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_crash.py +26 -2
  42. durablewrite-1.0.0/tests/test_crash_points.py +117 -0
  43. durablewrite-1.0.0/tests/test_docs.py +90 -0
  44. durablewrite-1.0.0/tests/test_e2e_upgrade.py +79 -0
  45. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_e2e_wheel.py +3 -0
  46. durablewrite-1.0.0/tests/test_fuzz.py +173 -0
  47. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_hardening.py +18 -11
  48. durablewrite-1.0.0/tests/test_linux_trace.py +158 -0
  49. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_metadata.py +7 -1
  50. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_options.py +2 -2
  51. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_overwrite.py +6 -2
  52. durablewrite-1.0.0/tests/test_packaging.py +41 -0
  53. durablewrite-1.0.0/tests/test_reference.py +107 -0
  54. durablewrite-1.0.0/tests/test_sources.py +41 -0
  55. durablewrite-1.0.0/tests/trace/dwtrace.c +180 -0
  56. durablewrite-0.2.0/.gitattributes +0 -1
  57. durablewrite-0.2.0/.github/FUNDING.yml +0 -1
  58. durablewrite-0.2.0/CHANGELOG.md +0 -66
  59. durablewrite-0.2.0/CONTRIBUTING.md +0 -24
  60. durablewrite-0.2.0/GUIDE.md +0 -207
  61. durablewrite-0.2.0/IMPROVEMENTS-round.md +0 -93
  62. durablewrite-0.2.0/PKG-INFO +0 -171
  63. durablewrite-0.2.0/README.md +0 -144
  64. durablewrite-0.2.0/requirements-dev.txt +0 -3
  65. durablewrite-0.2.0/scripts/check.ps1 +0 -11
  66. durablewrite-0.2.0/scripts/release.ps1 +0 -28
  67. durablewrite-0.2.0/src/durablewrite/_atomic.py +0 -141
  68. durablewrite-0.2.0/src/durablewrite/_options.py +0 -79
  69. {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/__main__.py +0 -0
  70. {durablewrite-0.2.0 → durablewrite-1.0.0}/src/durablewrite/py.typed +0 -0
  71. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/_payload.py +0 -0
  72. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/conftest.py +0 -0
  73. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_atomic.py +0 -0
  74. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_cli.py +0 -0
  75. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_contract.py +0 -0
  76. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_json_cleanup.py +0 -0
  77. {durablewrite-0.2.0 → durablewrite-1.0.0}/tests/test_sharing.py +0 -0
@@ -10,3 +10,5 @@ dist/
10
10
  .ruff_cache/
11
11
  .coverage
12
12
  htmlcov/
13
+ tests/oracle/*
14
+ !tests/oracle/MANIFEST.json
@@ -0,0 +1,139 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
5
+ [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
+
7
+ ## [Unreleased]
8
+
9
+ ## [1.0.0] - 2026-10-16
10
+
11
+ The public API is now stable: the twelve names in `durablewrite.__all__`, their
12
+ parameters, defaults and error types, the temp file naming scheme that
13
+ `cleanup_temp_files` relies on, and the command line options and exit statuses
14
+ follow Semantic Versioning from here on.
15
+
16
+ ### Changed
17
+
18
+ - `fsync="full"` (the default) on Windows renames with `MoveFileExW` and
19
+ `MOVEFILE_WRITE_THROUGH`, so the rename itself is written to disk before the
20
+ call returns; `"file"` and `"none"` keep plain `os.replace`.
21
+ - On macOS every flush uses `F_FULLFSYNC`, falling back to `fsync` where the
22
+ filesystem refuses it.
23
+ - All arguments are checked when a writer is called, before any file is
24
+ touched, instead of when the `with` block is entered.
25
+ - Error messages no longer repeat the refused value; the command line quotes at
26
+ most 40 characters of the values it checks and cuts every error or warning
27
+ message after its prefix to 120 characters, keeping its start and its end.
28
+ - On/off options (`overwrite`, `durable`, `fsync_dir`, `preserve_mode`,
29
+ `preserve_owner`, `follow_symlinks`) accept only `True` or `False`, and a `mode`
30
+ or `fsync` that is not a string raises `TypeError`: before, any value was taken
31
+ by truthiness, so `overwrite="no"` overwrote the file.
32
+ - `atomic_write` is typed per mode: `IO[str]` for `"w"`, `IO[bytes]` for `"wb"`
33
+ (`IO[Any]` only for a mode that is not a literal).
34
+ - `cleanup_temp_files` also removes read-only temp files on Windows, and
35
+ `older_than=0` removes every matching temp file whatever its time (on Windows
36
+ with Python 3.10 to 3.12 a file just written could look newer than the clock
37
+ and be skipped).
38
+ - Errors raised before the block runs name the path as it was given (for a
39
+ followed symlink, the file it points to); errors from
40
+ the rename name the target as an absolute path, since a relative target is
41
+ pinned to the working directory when the write starts.
42
+ - A `retry` that is not a `RetryPolicy` raises `TypeError` (0.2.0 raised
43
+ `AttributeError` and left a temp file).
44
+ - The `dev` extra is replaced by `requirements-dev.txt`.
45
+
46
+ ### Fixed
47
+
48
+ - `cleanup_temp_files(older_than=float("nan"))` removed every temp file,
49
+ including those of running writers; a NaN or infinite age, a non-finite `now`
50
+ and non-number ages now raise.
51
+ - `RetryPolicy` accepted NaN and infinite delays, which made a blocked write
52
+ fail with an unrelated error and leave its temp file behind, non-integer or
53
+ `bool` attempts and a `sleep` that is not callable; these now raise
54
+ `TypeError` or `ValueError`.
55
+ - A FIFO, device or socket target was silently replaced by a regular file
56
+ (`durablewrite /dev/null` as root replaced the device; `--append` on a FIFO
57
+ hung); it now raises `OSError` with `EINVAL`.
58
+ - Permission bits were set on the temp file by name; on POSIX they now go
59
+ through the descriptor (`fchmod`), so a temp name swapped for a symlink cannot
60
+ redirect them.
61
+ - A text wrapper that failed to build closed the file descriptor twice and hid
62
+ its own error behind `EBADF`; codecs that are not text encodings (`hex`) are now
63
+ refused up front with `LookupError`.
64
+ - An interrupt between creating the temp file and wrapping it left the
65
+ descriptor open, so on Windows the temp file could not be removed.
66
+ - `--json` refused valid JSON with integers longer than 4300 digits.
67
+ - On Windows a name with `:` wrote to an alternate data stream and left an empty
68
+ file behind; it now raises `ValueError`.
69
+ - `RetryPolicy` accepted delays so long that `time.sleep` overflowed; delays are
70
+ now limited to one day.
71
+ - `bytes` paths, paths containing NUL, `permissions` outside `0..0o7777`,
72
+ `permissions=True`, unknown encodings and invalid `newline` values failed late
73
+ or with unrelated errors; they now raise `TypeError`, `ValueError` or `LookupError` up front.
74
+
75
+ ### Added
76
+
77
+ - Ground truth in the test suite: kill-at-every-call tests on every platform, a
78
+ Linux system call tracer, 120 scenarios recorded from `atomicwrites` 1.4.1 on
79
+ Windows and Linux, and a manifest of the cited OS documentation.
80
+ - Doctests for every public name, seeded fuzz tests and linear-time checks.
81
+
82
+ ## [0.2.0] - 2026-10-14
83
+
84
+ ### Added
85
+
86
+ - `write_json_atomic(path, obj, ...)`: serialises before touching the disk and
87
+ writes UTF-8 JSON with a trailing newline and LF line endings. `NaN` and
88
+ infinities are refused unless `allow_nan=True`.
89
+ - `fsync="none" | "file" | "full"` on every writer, with the `FsyncLevel` type.
90
+ The `durable` and `fsync_dir` flags keep working.
91
+ - `overwrite=False`: atomic create-if-absent (`link` on POSIX, `MoveFileExW`
92
+ without the replace flag on Windows).
93
+ - `preserve_mode` (default on), `permissions` and `preserve_owner` options.
94
+ - `follow_symlinks` (default on): writing through a symlink updates the file it
95
+ points to and keeps the link.
96
+ - `RetryPolicy`, `DEFAULT_RETRY` and `NO_RETRY`: bounded backoff when a Windows
97
+ sharing or lock violation blocks the rename.
98
+ - `DurableWriteError` (an `OSError`), `SharingViolationError` (a
99
+ `PermissionError`) and `DirectorySyncError` (the file was replaced but its
100
+ directory could not be fsynced).
101
+ - `cleanup_temp_files(directory, older_than=..., target=...)` for temp files
102
+ left by killed writers.
103
+ - `durablewrite` command line tool (also `python -m durablewrite`): writes
104
+ standard input to a file atomically, with `--append`, `--json`, `--fsync`,
105
+ `--no-overwrite`, `--permissions`, `--no-follow-symlinks` and `--attempts`.
106
+ Exit status 1 on errors, 3 when the file was replaced but not fully synced.
107
+ - `GUIDE.md` with the write protocol, per-platform guarantees and sources.
108
+
109
+ ### Changed
110
+
111
+ - Temp files are named `.<name>.<8 hex digits>.dwtmp` so they can be cleaned up
112
+ safely.
113
+ - Writing through a symlink no longer replaces the link with a regular file
114
+ (pass `follow_symlinks=False` for the old behaviour).
115
+
116
+ ### Fixed
117
+
118
+ - New files were created with mode `0600` and replaced files lost their mode;
119
+ new files now honour the umask like `open()` and replaced files keep their
120
+ permission bits.
121
+ - Very long file names no longer overflow the name limit when the temp name is
122
+ built.
123
+ - Writing to an existing directory raises `IsADirectoryError` before any temp
124
+ file is created, and a path ending with a separator raises `ValueError`.
125
+ - When the final mode is restricted (explicit `permissions` or a replaced
126
+ private file) the temp file is `0600` while it is written.
127
+ - On Windows, removal of a temp file that is briefly locked is retried.
128
+
129
+ ## [0.1.0] - 2026-10-11 (not published)
130
+
131
+ ### Added
132
+
133
+ - `atomic_write(path, mode, *, encoding, newline, durable, fsync_dir)` context
134
+ manager: writes to a temp file in the target directory, flushes and fsyncs it,
135
+ then moves it onto the target with `os.replace` and fsyncs the directory.
136
+ - `write_text_atomic(path, text, encoding, *, newline, durable, fsync_dir)` and
137
+ `write_bytes_atomic(path, data, *, durable, fsync_dir)` one-shot helpers.
138
+ - Durable by default (file and directory fsync), both switchable off.
139
+ - Correct on Windows and POSIX; typed, zero runtime dependencies.
@@ -0,0 +1,29 @@
1
+ # Contributing
2
+
3
+ Thanks for taking the time. This project keeps a small, predictable shape.
4
+
5
+ ## Setup
6
+
7
+ ```
8
+ pip install -r requirements-dev.txt
9
+ python scripts/fetch_oracles.py # optional: reference library and cited pages
10
+ ```
11
+
12
+ ## Before you open a pull request
13
+
14
+ - Keep public API in `src/durablewrite/__init__.py`; internals live in `_`-prefixed modules.
15
+ - Add a test for the behaviour you change. Tests run against the real filesystem and
16
+ real child processes; only the retry `sleep` and, in a few contract tests, single
17
+ OS calls are replaced to observe what is asked of the operating system.
18
+ - Run the gate: `ruff format --check .`, `ruff check .`, `mypy --strict src tests` and
19
+ `pytest` (the repository has `scripts/check.ps1` for this). The multi-process and
20
+ packaging tests are marked `slow`; `pytest -m "not slow"` skips them while
21
+ iterating, but the full suite must pass before a commit. Run it on Windows and on
22
+ Linux; the system call tests need Linux and `gcc`.
23
+ - The crash and fuzz tests are seeded: set `DURABLEWRITE_CRASH_SEED`,
24
+ `DURABLEWRITE_CRASH_ITERATIONS` or `DURABLEWRITE_FUZZ_SEED` to explore further.
25
+ - Material fetched by `scripts/fetch_oracles.py` stays out of the repository; only
26
+ `tests/oracle/MANIFEST.json` is tracked.
27
+ - One logical change per commit, imperative subject line.
28
+
29
+ Open an issue first if the change is large or shifts the public API.
@@ -0,0 +1,309 @@
1
+ # durablewrite guide
2
+
3
+ This guide explains what `durablewrite` does on every write, what it guarantees
4
+ on each platform, and where those guarantees stop. Each claim names its source
5
+ or the test in this repository that exercises it.
6
+
7
+ ## The write protocol
8
+
9
+ For `atomic_write(path)` and every helper built on it:
10
+
11
+ 1. **Check the arguments** when the writer is called, before any file is
12
+ touched (see "Errors" below).
13
+ 2. **Resolve the target.** If `path` is a symlink and `follow_symlinks=True`
14
+ (the default), the file it points to becomes the target, so the link keeps
15
+ working. With `follow_symlinks=False` the link itself is replaced by a
16
+ regular file. An existing directory is rejected with `IsADirectoryError`, a
17
+ FIFO, device or socket with `OSError` (`EINVAL`) so it is never swapped for a
18
+ regular file, and
19
+ with `overwrite=False` an existing name is rejected with `FileExistsError`,
20
+ both before the block runs.
21
+ 3. **Create a temp file in the target's own directory**, named
22
+ `.<name>.<8 hex digits>.dwtmp`, with `O_CREAT | O_EXCL`. A new file is created
23
+ with mode `0o666`, so the process umask applies exactly as it does for
24
+ `open()`. When the final mode comes from `permissions` or from the replaced
25
+ file, the temp file starts as `0o600` and is widened only after the data is
26
+ written, so restricted content is never readable by others in between. Names
27
+ longer than 48 characters keep a readable head plus a hash of the full name,
28
+ so the temp name stays within the 255-byte limit. The same directory means
29
+ the same filesystem, which the rename needs: Python documents that
30
+ `os.replace` may fail across filesystems [1].
31
+ 4. **Write** your data into it.
32
+ 5. **Set metadata.** With `preserve_owner=True` (POSIX) the uid and gid of the
33
+ old file are copied first, then the permission bits: either `permissions` or,
34
+ with `preserve_mode=True` (the default), the bits of the old file. On POSIX
35
+ both go through the open descriptor (`fchown`, `fchmod`), so a temp name
36
+ swapped for a symlink by someone else cannot redirect them.
37
+ 6. **Flush the temp file to disk** (`fsync="file"` or `"full"`): `fsync` on
38
+ Linux, `F_FULLFSYNC` on macOS, `FlushFileBuffers` on Windows. This must come
39
+ before the rename, or the new name can reach the disk before the data it
40
+ points to, which is how ext4 left zero-length files after a crash until it
41
+ added a heuristic for that pattern [3][4].
42
+ 7. **Rename** the temp file onto the target. With `overwrite=False` a
43
+ create-only rename is used instead (see below).
44
+ 8. **Make the rename durable** (`fsync="full"`). On POSIX the parent directory is
45
+ fsynced: the Linux `fsync(2)` page says a file fsync does not by itself make
46
+ the directory entry durable, and asks for an fsync of the directory [2][3].
47
+ On Windows the rename of step 7 is made with `MOVEFILE_WRITE_THROUGH`, which
48
+ Microsoft documents as not returning until the move is on the disk [6].
49
+
50
+ If anything raises before step 7 completes, the temp file is removed and the
51
+ target is untouched. That includes exceptions inside your `with` block and
52
+ `KeyboardInterrupt`.
53
+
54
+ ## What is guaranteed, per platform
55
+
56
+ | Promise | Linux / macOS (local fs) | Windows (local NTFS) |
57
+ | --- | --- | --- |
58
+ | Readers see the old or the complete new file, never a mix | Yes, `rename(2)` replaces the name atomically [5][12] | Yes on NTFS in our tests; Microsoft does not call the rename atomic [6] |
59
+ | A killed writer leaves the old or the new file, never a partial one | Yes (tested at every call) | Yes (tested at every call) |
60
+ | New content survives power loss once the call returns (`"full"`) | Linux: data and directory fsynced. macOS: `F_FULLFSYNC` [8] | Data flushed [7]; rename written through [6] |
61
+ | Permission bits kept on replace | Yes (`preserve_mode`) | Not applicable; see below |
62
+ | Owner kept on replace | Opt-in (`preserve_owner`), needs privilege to change uid | No |
63
+ | Replace while another process has the target open | Yes; the reader keeps the old inode | Retried, then `SharingViolationError` |
64
+ | A reader can always open the target | Yes | No: an `open()` racing a rename can fail for a moment with `PermissionError` or, rarely, `FileNotFoundError` (never torn); readers that poll should retry both |
65
+ | Replace a read-only target | Yes, if the directory is writable | No, `PermissionError` (not retried) |
66
+
67
+ ### Process crash versus power loss
68
+
69
+ A process crash is tested directly. `tests/test_crash_points.py` runs a writer
70
+ that kills itself with no cleanup right before and right after each file system
71
+ call it makes, for every call, on Windows and POSIX. `tests/test_linux_trace.py`
72
+ does the same on Linux one level lower: an `LD_PRELOAD` library
73
+ (`tests/trace/dwtrace.c`) logs the real libc calls (`open`, `write`, `fsync`,
74
+ `rename`, `link`, `unlink`) and sends `SIGKILL` at each boundary. In every case
75
+ the target holds the old content until the rename has returned and the new
76
+ content from then on, and any leftover temp file is one that
77
+ `cleanup_temp_files` removes. `tests/test_crash.py` also kills writers at random
78
+ moments mid-write.
79
+
80
+ No portable test can cut the power, so durability against power loss rests on
81
+ the documented calls above and on the filesystem honouring them [3].
82
+
83
+ ### Linux
84
+
85
+ `fsync="full"` follows the sequence recommended in "Ensuring data reaches disk"
86
+ [3]: fsync the file, rename, fsync the directory. The trace test records exactly
87
+ that order. Do not rely on ext4's `auto_da_alloc` heuristic for "replace via
88
+ rename" [4]; it is an ext4 mount option that can be turned off, not a promise of
89
+ the rename itself. If the directory fsync raises (some network and FUSE filesystems refuse it,
90
+ and it fails when the directory is writable but not readable), the new content is
91
+ already in place and `DirectorySyncError` is raised so you know the rename may
92
+ not be durable (`tests/test_hardening.py`).
93
+
94
+ ### macOS
95
+
96
+ Apple documents that `fsync` hands the data to the drive without asking the
97
+ drive to empty its own write cache, and offers the `F_FULLFSYNC` fcntl for that
98
+ [8]. `durablewrite` issues `F_FULLFSYNC` for the file and for the directory, and
99
+ falls back to `fsync` on a filesystem that refuses it. This is the same choice
100
+ `atomicwrites` made. macOS was not part of this release's test runs; the code
101
+ path is pinned by `tests/test_contract_platform.py`.
102
+
103
+ ### Windows
104
+
105
+ - `os.fsync` calls the C runtime's `_commit` [1][9], which forces the file to
106
+ disk; on Windows 11 a hook on the C runtime showed `_commit` calling
107
+ `FlushFileBuffers` [7] (observed, not documented on the `_commit` page).
108
+ - With `fsync="full"`, the rename is `MoveFileExW` with
109
+ `MOVEFILE_WRITE_THROUGH` (plus `MOVEFILE_REPLACE_EXISTING` unless
110
+ `overwrite=False`) [6], as `atomicwrites` does. With `"file"` or `"none"` it is
111
+ plain `os.replace`, which is `MoveFileExW` without write-through.
112
+ - There is no directory fsync through the APIs Python exposes; write-through
113
+ takes its place.
114
+ - Microsoft does not document the rename as atomic. On a local NTFS volume it is
115
+ a single metadata operation and our tests never observed a torn target. A
116
+ reader opening the name while it is being replaced can, rarely, get
117
+ `FileNotFoundError` (about 1 open in 1,000 to 2,000 with four writers
118
+ hammering one file); 0.2.0 behaved the same. On FAT volumes and SMB shares the behaviour depends on the filesystem
119
+ or server and is not guaranteed.
120
+ - The rename fails with `ERROR_ACCESS_DENIED`, `ERROR_SHARING_VIOLATION` or
121
+ `ERROR_LOCK_VIOLATION` while any process holds the target open without
122
+ `FILE_SHARE_DELETE` [10]. Python's `open()` does not pass that flag, and
123
+ antivirus scanners and the search indexer briefly open new files. These errors
124
+ are retried with bounded backoff (`RetryPolicy`, about 1.1 seconds by default).
125
+ A read-only target is never retried. Several processes replacing one file
126
+ while others read it can keep it locked for longer than that on a slow or busy
127
+ machine, and so can many writers of one file on their own (an adversarial run
128
+ of 16 threads saw a few blocked renames in 3,200); give such writers a larger
129
+ `RetryPolicy`. When the budget runs out,
130
+ `SharingViolationError` is raised and the target is untouched
131
+ (`tests/test_sharing.py`). A directory ACL that denies the rename produces the
132
+ same error code and ends the same way.
133
+ - The temp file can itself be held open (an antivirus scan of the fresh file).
134
+ On every error path its removal is retried under the same `RetryPolicy`; if it
135
+ is still locked after that, it is left for `cleanup_temp_files`, which also
136
+ clears a read-only attribute before removing.
137
+ - Permission bits do not exist on Windows. `permissions` only toggles the
138
+ read-only attribute and `preserve_mode` does nothing. The new file inherits the
139
+ ACL of its directory; an explicit ACL on the old file is not copied.
140
+
141
+ ### Why not `ReplaceFileW`
142
+
143
+ `ReplaceFileW` [11] keeps the replaced file's ACL, attributes and creation time,
144
+ but it is built around an existing file to replace (in our tests on Windows 11
145
+ it failed when the target was missing) and it can fail half way: Microsoft lists a
146
+ result, `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2`, in which the replaced file is left
147
+ under another name. A primitive whose failure mode removes the target conflicts
148
+ with the core promise, so `durablewrite` uses `MoveFileExW`.
149
+
150
+ ## Create only if absent
151
+
152
+ `overwrite=False` publishes the complete file only if nothing exists at the
153
+ target, and raises `FileExistsError` otherwise, without a check-then-act race:
154
+
155
+ - POSIX: `link(temp, target)` fails with `EEXIST` if the name is taken [13], then
156
+ the temp name is unlinked. The filesystem must support hard links.
157
+ - Windows: `MoveFileExW` without `MOVEFILE_REPLACE_EXISTING` fails with error
158
+ 183 (`FileExistsError`), observed on NTFS (`tests/test_contract_platform.py`).
159
+
160
+ An existing name is refused before the block runs; a name that appears while
161
+ the block runs is refused by the rename. `tests/test_overwrite.py` races six
162
+ processes for one name and checks that exactly one wins.
163
+
164
+ ## Symlinks
165
+
166
+ The default (`follow_symlinks=True`) writes to the file the link points to and
167
+ creates the temp file beside that file, so the rename never crosses a
168
+ filesystem even when the link lives elsewhere. A dangling link creates its
169
+ target. `follow_symlinks=False` replaces the link with a regular file, which is
170
+ what plain `os.replace` (and `atomicwrites`) does; on POSIX this also works when
171
+ the link points to a directory. The replaced link counts as a new file, so it
172
+ does not inherit the mode of whatever it pointed to.
173
+
174
+ ## Side effects of replacing a file
175
+
176
+ Replacing a file creates a new file, by design. As a consequence:
177
+
178
+ - Other hard links to the old file keep the old content.
179
+ - The inode (POSIX) or file ID (Windows) changes. Programs that follow a file by
180
+ handle, such as `tail -f` without `-F`, keep reading the old file.
181
+ - Extended attributes, POSIX ACLs and Windows alternate data streams of the old
182
+ file are not copied.
183
+
184
+ ## Leftover temp files
185
+
186
+ A writer that is killed between creating its temp file and renaming it cannot
187
+ clean up. `cleanup_temp_files(directory, older_than=3600)` removes only regular
188
+ files matching `.<name>.<8 hex digits>.dwtmp` older than the given age, by
189
+ modification time. A writer that keeps its temp file open for longer than that
190
+ age could lose it, so choose an age longer than your slowest write. With
191
+ `target=`, only temp files of that exact target name are considered. The age
192
+ must be a finite number of seconds, not negative: `NaN` once made every temp
193
+ file look stale, so it is refused.
194
+
195
+ ## Errors
196
+
197
+ | Error | When |
198
+ | --- | --- |
199
+ | `TypeError` | a `bytes` path, an on/off option that is not `True` or `False`, or another argument of the wrong type |
200
+ | `ValueError` | a bad `mode`, `fsync`, `newline`, `permissions`, retry setting (delays from 0 to one day) or age; an empty path, a NUL in it, one ending with a separator, or on Windows a name with `:` (an alternate data stream) |
201
+ | `LookupError` | an unknown `encoding`, or a codec that is not a text encoding (`hex`) |
202
+ | `IsADirectoryError` | the target is a directory |
203
+ | `OSError` (`EINVAL`) | the target is a FIFO, device or socket |
204
+ | `FileExistsError` | `overwrite=False` and the name exists |
205
+ | `SharingViolationError` | Windows kept the rename blocked for the whole retry budget (a `PermissionError`) |
206
+ | `DirectorySyncError` | the file was replaced, but the directory fsync failed (POSIX, `"full"`) |
207
+ | other `OSError` | whatever the operating system reports: a missing directory, no space, no permission |
208
+
209
+ The first three are raised when the writer is called, before any file is
210
+ touched. Messages are short and never repeat the value they refuse.
211
+ `DurableWriteError` is the base of the two errors `durablewrite` raises itself
212
+ and is an `OSError`, so existing `except OSError` code keeps working.
213
+
214
+ ## Command line
215
+
216
+ `durablewrite TARGET` copies standard input into TARGET with the protocol above,
217
+ like `sponge` from moreutils. TARGET is replaced only after standard input ends,
218
+ so `sort f | durablewrite f` is safe on every platform, and
219
+ `durablewrite --append f < f` works on POSIX. On Windows the shell (and the
220
+ console-script launcher) keep a redirected file open without
221
+ `FILE_SHARE_DELETE`, which blocks the rename, so the command detects that
222
+ standard input is TARGET and refuses at once; pipe the file instead.
223
+
224
+ Exit status: 0 on success; 1 on any error, including a usage error, with TARGET
225
+ left as it was; 3 when TARGET was replaced but its directory could not be
226
+ fsynced; 130 when interrupted. `--json` accepts only strict JSON (no `NaN` or
227
+ `Infinity`). Each `--append` run is atomic, but two concurrent runs on the same
228
+ file can lose one run's data, because each starts from the content it read.
229
+
230
+ ## How this was checked
231
+
232
+ - **Documentation**: `tests/oracle/MANIFEST.json` lists every page cited below
233
+ with its URL, fetch date, size and hash; `scripts/fetch_oracles.py` downloads
234
+ them into a local folder that is never committed, and `tests/test_sources.py`
235
+ checks that each still names the calls and flags this guide relies on.
236
+ - **System calls**: `tests/test_linux_trace.py` (Linux, needs gcc) records the
237
+ real call order for each fsync level and kills the writer at every call.
238
+ - **Reference library**: `atomicwrites` 1.4.1 ran 120 scenarios (new file,
239
+ replace, create-only, directory, symlink, dangling link, failing block, text and
240
+ binary) on Windows and Linux; `tests/fixtures/atomicwrites-*.json` is its
241
+ recorded output, and `tests/test_reference.py` checks that durablewrite matches
242
+ it wherever a plain file is written, and differs only as this guide says
243
+ elsewhere. Under the tracer both libraries make the same fsync, rename,
244
+ directory fsync sequence.
245
+
246
+ ## Upgrading from 0.2
247
+
248
+ Code that passes the documented argument types keeps working; the differences:
249
+
250
+ - `fsync="full"` now uses `MOVEFILE_WRITE_THROUGH` on Windows and `F_FULLFSYNC`
251
+ on macOS.
252
+ - Arguments are checked when the writer is called rather than when the `with`
253
+ block is entered, and more of them are checked: `bytes` paths, paths with a
254
+ NUL, `permissions` outside `0..0o7777`, unknown encodings, invalid `newline`
255
+ values, codecs that are not text encodings, a `retry` that is not a
256
+ `RetryPolicy`, and on Windows alternate data stream names.
257
+ - A FIFO, device or socket target raises `OSError` (`EINVAL`); 0.2.0 replaced it
258
+ with a regular file (as root, `durablewrite /dev/null` replaced the device).
259
+ - `RetryPolicy` delays longer than one day are refused.
260
+ - `--json` checks numbers without converting them, so integers longer than 4300
261
+ digits are accepted.
262
+ - On/off options (`overwrite`, `durable`, `fsync_dir`, `preserve_mode`,
263
+ `preserve_owner`, `follow_symlinks`) must be `True` or `False`: 0.2.0 took any
264
+ value by truthiness, so `overwrite="no"` overwrote. A `mode` or `fsync` that is
265
+ not a string, and `permissions=True`, raise `TypeError`.
266
+ - `RetryPolicy` refuses attempts that are not an `int` (a `bool` included), a
267
+ `sleep` that is not callable, and delays or multipliers that are NaN, infinite
268
+ or of the wrong type; `cleanup_temp_files` refuses a NaN or
269
+ infinite age and a non-finite `now`.
270
+ - `cleanup_temp_files` removes read-only temp files on Windows, and
271
+ `older_than=0` removes every matching temp file whatever its time.
272
+ - Errors raised before the block runs name the path as given (for a followed
273
+ symlink, the file it points to); errors from the rename name the absolute
274
+ target (a relative target is pinned when the write
275
+ starts).
276
+ - Error messages no longer repeat the refused value. The command line quotes at
277
+ most 40 characters of the values it checks itself, and every error or warning
278
+ message it prints after its `durablewrite: error:` prefix is cut to 120
279
+ characters, keeping its start and its end (where the file name is).
280
+ - `atomic_write` is typed per mode: `IO[str]` for `"w"`, `IO[bytes]` for `"wb"`.
281
+ - The `dev` extra is gone; contributors use `requirements-dev.txt`.
282
+
283
+ ## Sources
284
+
285
+ Fetched on 2026-10-16.
286
+
287
+ 1. Python documentation, `os.replace` and `os.fsync`:
288
+ https://docs.python.org/3/library/os.html
289
+ 2. Linux `fsync(2)` manual page: https://man7.org/linux/man-pages/man2/fsync.2.html
290
+ 3. Jeff Moyer, "Ensuring data reaches disk", LWN.net, 2011:
291
+ https://lwn.net/Articles/457667/
292
+ 4. Linux kernel documentation, ext4 mount options (`auto_da_alloc`):
293
+ https://docs.kernel.org/admin-guide/ext4.html
294
+ 5. POSIX.1-2024, `rename`:
295
+ https://pubs.opengroup.org/onlinepubs/9799919799/functions/rename.html
296
+ 6. Microsoft, `MoveFileExW`:
297
+ https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-movefileexw
298
+ 7. Microsoft, `FlushFileBuffers`:
299
+ https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-flushfilebuffers
300
+ 8. Apple, `fsync(2)` manual page (`F_FULLFSYNC`):
301
+ https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fsync.2.html
302
+ 9. Microsoft, `_commit`:
303
+ https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/commit
304
+ 10. Microsoft, `CreateFileW` (`FILE_SHARE_DELETE`):
305
+ https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
306
+ 11. Microsoft, `ReplaceFileW`:
307
+ https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-replacefilew
308
+ 12. Linux `rename(2)` manual page: https://man7.org/linux/man-pages/man2/rename.2.html
309
+ 13. Linux `link(2)` manual page: https://man7.org/linux/man-pages/man2/link.2.html
@@ -1,21 +1,21 @@
1
- MIT License
2
-
3
- Copyright (c) 2026 Javier Lianes
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining a copy
6
- of this software and associated documentation files (the "Software"), to deal
7
- in the Software without restriction, including without limitation the rights
8
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
- copies of the Software, and to permit persons to whom the Software is
10
- furnished to do so, subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in all
13
- copies or substantial portions of the Software.
14
-
15
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
- SOFTWARE.
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Javier Lianes
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.