kitsuyui.fileio 0.8.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kitsuyui_fileio-0.8.1/LICENSE +11 -0
- kitsuyui_fileio-0.8.1/PKG-INFO +84 -0
- kitsuyui_fileio-0.8.1/README.md +71 -0
- kitsuyui_fileio-0.8.1/pyproject.toml +37 -0
- kitsuyui_fileio-0.8.1/setup.cfg +4 -0
- kitsuyui_fileio-0.8.1/src/kitsuyui/fileio/__init__.py +17 -0
- kitsuyui_fileio-0.8.1/src/kitsuyui/fileio/_atomic.py +209 -0
- kitsuyui_fileio-0.8.1/src/kitsuyui/fileio/_version.py +24 -0
- kitsuyui_fileio-0.8.1/src/kitsuyui/fileio/py.typed +0 -0
- kitsuyui_fileio-0.8.1/src/kitsuyui.fileio.egg-info/PKG-INFO +84 -0
- kitsuyui_fileio-0.8.1/src/kitsuyui.fileio.egg-info/SOURCES.txt +14 -0
- kitsuyui_fileio-0.8.1/src/kitsuyui.fileio.egg-info/dependency_links.txt +1 -0
- kitsuyui_fileio-0.8.1/src/kitsuyui.fileio.egg-info/scm_file_list.json +11 -0
- kitsuyui_fileio-0.8.1/src/kitsuyui.fileio.egg-info/scm_version.json +8 -0
- kitsuyui_fileio-0.8.1/src/kitsuyui.fileio.egg-info/top_level.txt +1 -0
- kitsuyui_fileio-0.8.1/tests/test_atomic.py +235 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Copyright 2023-2025 Kitsu Yui
|
|
2
|
+
|
|
3
|
+
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
|
4
|
+
|
|
5
|
+
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
|
6
|
+
|
|
7
|
+
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
|
8
|
+
|
|
9
|
+
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
|
10
|
+
|
|
11
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kitsuyui.fileio
|
|
3
|
+
Version: 0.8.1
|
|
4
|
+
Summary: Explicit and safe file writing utilities
|
|
5
|
+
Author-email: Yui KITSU <kitsuyui+github@kitsuyui.com>
|
|
6
|
+
License-Expression: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://github.com/kitsuyui/pypi-playground
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Dynamic: license-file
|
|
13
|
+
|
|
14
|
+
# kitsuyui.fileio
|
|
15
|
+
|
|
16
|
+
Small, explicit file I/O primitives. The first release provides atomic
|
|
17
|
+
replacement of a single file without runtime dependencies (Python 3.10+).
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
from kitsuyui.fileio import atomic_write, atomic_write_bytes, atomic_write_text
|
|
21
|
+
|
|
22
|
+
atomic_write_text("settings.txt", "ready\n")
|
|
23
|
+
atomic_write_bytes("data.bin", b"\x00\x01")
|
|
24
|
+
with atomic_write("large.bin") as output:
|
|
25
|
+
output.write(b"first chunk")
|
|
26
|
+
output.write(b"second chunk")
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
`atomic_write` yields a binary stream. On normal exit it flushes and closes the
|
|
30
|
+
stream before replacing the destination. An exception inside the block leaves
|
|
31
|
+
the destination untouched and removes the temporary file when possible. Do not
|
|
32
|
+
close the yielded stream manually. The convenience functions accept paths and
|
|
33
|
+
bytes or text respectively; text is encoded as UTF-8 unless specified otherwise.
|
|
34
|
+
The parent directory must already exist. A caller that wraps the binary stream
|
|
35
|
+
in a `TextIOWrapper` must flush and detach the wrapper before leaving the block;
|
|
36
|
+
closing the supplied stream causes a `ValueError` and cancels replacement.
|
|
37
|
+
Text convenience writes use `str.encode` and do not translate newlines; callers
|
|
38
|
+
may choose the encoding and encoding error handler.
|
|
39
|
+
|
|
40
|
+
## Guarantees and boundaries
|
|
41
|
+
|
|
42
|
+
The temporary file is created in the destination directory and installed with
|
|
43
|
+
`os.replace`. On a filesystem supporting POSIX atomic rename, a reader opening
|
|
44
|
+
the destination sees either the previous file or the complete new file during
|
|
45
|
+
normal execution; an
|
|
46
|
+
already-open descriptor may continue to refer to the old file. On other systems
|
|
47
|
+
and filesystems, replacement guarantees depend on the platform. This does not
|
|
48
|
+
lock writers, prevent lost updates, make multiple replacements transactional, or
|
|
49
|
+
secure a directory controlled by an untrusted party. Concurrent successful
|
|
50
|
+
writers are last-replacement-wins. A destination symlink itself is replaced;
|
|
51
|
+
its target is not modified. The temporary file is created with restrictive
|
|
52
|
+
permissions (0600 on POSIX). Existing ownership, permissions, timestamps, ACLs,
|
|
53
|
+
extended attributes and hard links are not preserved. The restrictive default
|
|
54
|
+
avoids silently exposing content from a private file or temporary write. To
|
|
55
|
+
choose another POSIX permission mode, pass `permissions=0o644` (for example);
|
|
56
|
+
the specified bits are applied to the temporary file before replacement and
|
|
57
|
+
are not filtered by umask. Do not use permissive modes for sensitive data. On
|
|
58
|
+
Windows `os.chmod` has limited permission semantics. Copying existing permissions
|
|
59
|
+
is deliberately left to the caller because metadata and access policy are
|
|
60
|
+
platform-specific and can change concurrently.
|
|
61
|
+
|
|
62
|
+
The keyword `durability` distinguishes visibility from persistence:
|
|
63
|
+
|
|
64
|
+
* `"none"`: flush and close before replacement; no `fsync`.
|
|
65
|
+
* `"file"` (default): also `fsync` the staged file before replacement.
|
|
66
|
+
* `"file_and_directory"`: also `fsync` the parent directory after replacement.
|
|
67
|
+
This option is available only on POSIX and can still fail on a filesystem
|
|
68
|
+
that does not support directory sync.
|
|
69
|
+
|
|
70
|
+
Sync calls request persistence from the operating system; they cannot guarantee
|
|
71
|
+
the behavior of every filesystem, storage device, or hardware buffer. A failure
|
|
72
|
+
before `os.replace` leaves an existing destination unchanged. If directory
|
|
73
|
+
sync fails after replacement, `DurabilityError` is raised: the new destination
|
|
74
|
+
is already visible, but its persistence is uncertain. If replacement itself fails,
|
|
75
|
+
the operating system error is propagated. Abrupt process termination can leave a
|
|
76
|
+
temporary file behind; ordinary exception cleanup is best effort. A crash with
|
|
77
|
+
`"none"` can leave the new file empty or truncated even after the rename, so
|
|
78
|
+
the runtime visibility guarantee is not a crash-recovery guarantee. On macOS,
|
|
79
|
+
`os.fsync` does not request the stronger `F_FULLFSYNC` operation, so even a
|
|
80
|
+
successful sync does not guarantee that hardware buffers reached durable media.
|
|
81
|
+
|
|
82
|
+
## License
|
|
83
|
+
|
|
84
|
+
BSD 3-Clause License. See the repository's LICENSE.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# kitsuyui.fileio
|
|
2
|
+
|
|
3
|
+
Small, explicit file I/O primitives. The first release provides atomic
|
|
4
|
+
replacement of a single file without runtime dependencies (Python 3.10+).
|
|
5
|
+
|
|
6
|
+
```python
|
|
7
|
+
from kitsuyui.fileio import atomic_write, atomic_write_bytes, atomic_write_text
|
|
8
|
+
|
|
9
|
+
atomic_write_text("settings.txt", "ready\n")
|
|
10
|
+
atomic_write_bytes("data.bin", b"\x00\x01")
|
|
11
|
+
with atomic_write("large.bin") as output:
|
|
12
|
+
output.write(b"first chunk")
|
|
13
|
+
output.write(b"second chunk")
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
`atomic_write` yields a binary stream. On normal exit it flushes and closes the
|
|
17
|
+
stream before replacing the destination. An exception inside the block leaves
|
|
18
|
+
the destination untouched and removes the temporary file when possible. Do not
|
|
19
|
+
close the yielded stream manually. The convenience functions accept paths and
|
|
20
|
+
bytes or text respectively; text is encoded as UTF-8 unless specified otherwise.
|
|
21
|
+
The parent directory must already exist. A caller that wraps the binary stream
|
|
22
|
+
in a `TextIOWrapper` must flush and detach the wrapper before leaving the block;
|
|
23
|
+
closing the supplied stream causes a `ValueError` and cancels replacement.
|
|
24
|
+
Text convenience writes use `str.encode` and do not translate newlines; callers
|
|
25
|
+
may choose the encoding and encoding error handler.
|
|
26
|
+
|
|
27
|
+
## Guarantees and boundaries
|
|
28
|
+
|
|
29
|
+
The temporary file is created in the destination directory and installed with
|
|
30
|
+
`os.replace`. On a filesystem supporting POSIX atomic rename, a reader opening
|
|
31
|
+
the destination sees either the previous file or the complete new file during
|
|
32
|
+
normal execution; an
|
|
33
|
+
already-open descriptor may continue to refer to the old file. On other systems
|
|
34
|
+
and filesystems, replacement guarantees depend on the platform. This does not
|
|
35
|
+
lock writers, prevent lost updates, make multiple replacements transactional, or
|
|
36
|
+
secure a directory controlled by an untrusted party. Concurrent successful
|
|
37
|
+
writers are last-replacement-wins. A destination symlink itself is replaced;
|
|
38
|
+
its target is not modified. The temporary file is created with restrictive
|
|
39
|
+
permissions (0600 on POSIX). Existing ownership, permissions, timestamps, ACLs,
|
|
40
|
+
extended attributes and hard links are not preserved. The restrictive default
|
|
41
|
+
avoids silently exposing content from a private file or temporary write. To
|
|
42
|
+
choose another POSIX permission mode, pass `permissions=0o644` (for example);
|
|
43
|
+
the specified bits are applied to the temporary file before replacement and
|
|
44
|
+
are not filtered by umask. Do not use permissive modes for sensitive data. On
|
|
45
|
+
Windows `os.chmod` has limited permission semantics. Copying existing permissions
|
|
46
|
+
is deliberately left to the caller because metadata and access policy are
|
|
47
|
+
platform-specific and can change concurrently.
|
|
48
|
+
|
|
49
|
+
The keyword `durability` distinguishes visibility from persistence:
|
|
50
|
+
|
|
51
|
+
* `"none"`: flush and close before replacement; no `fsync`.
|
|
52
|
+
* `"file"` (default): also `fsync` the staged file before replacement.
|
|
53
|
+
* `"file_and_directory"`: also `fsync` the parent directory after replacement.
|
|
54
|
+
This option is available only on POSIX and can still fail on a filesystem
|
|
55
|
+
that does not support directory sync.
|
|
56
|
+
|
|
57
|
+
Sync calls request persistence from the operating system; they cannot guarantee
|
|
58
|
+
the behavior of every filesystem, storage device, or hardware buffer. A failure
|
|
59
|
+
before `os.replace` leaves an existing destination unchanged. If directory
|
|
60
|
+
sync fails after replacement, `DurabilityError` is raised: the new destination
|
|
61
|
+
is already visible, but its persistence is uncertain. If replacement itself fails,
|
|
62
|
+
the operating system error is propagated. Abrupt process termination can leave a
|
|
63
|
+
temporary file behind; ordinary exception cleanup is best effort. A crash with
|
|
64
|
+
`"none"` can leave the new file empty or truncated even after the rename, so
|
|
65
|
+
the runtime visibility guarantee is not a crash-recovery guarantee. On macOS,
|
|
66
|
+
`os.fsync` does not request the stronger `F_FULLFSYNC` operation, so even a
|
|
67
|
+
successful sync does not guarantee that hardware buffers reached durable media.
|
|
68
|
+
|
|
69
|
+
## License
|
|
70
|
+
|
|
71
|
+
BSD 3-Clause License. See the repository's LICENSE.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "kitsuyui.fileio"
|
|
3
|
+
dynamic = ["version"]
|
|
4
|
+
description = "Explicit and safe file writing utilities"
|
|
5
|
+
license = "BSD-3-Clause"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
readme = { file = "README.md", content-type = "text/markdown" }
|
|
8
|
+
authors = [{name = "Yui KITSU", email = "kitsuyui+github@kitsuyui.com"}]
|
|
9
|
+
classifiers = ["Development Status :: 3 - Alpha"]
|
|
10
|
+
|
|
11
|
+
[project.urls]
|
|
12
|
+
Homepage = "https://github.com/kitsuyui/pypi-playground"
|
|
13
|
+
|
|
14
|
+
[build-system]
|
|
15
|
+
requires = ["setuptools>=77", "setuptools_scm"]
|
|
16
|
+
build-backend = "setuptools.build_meta"
|
|
17
|
+
|
|
18
|
+
[tool.setuptools]
|
|
19
|
+
package-dir = { "" = "src" }
|
|
20
|
+
package-data = { "*" = ["py.typed"] }
|
|
21
|
+
|
|
22
|
+
[tool.setuptools.packages.find]
|
|
23
|
+
where = ["src/"]
|
|
24
|
+
include = ["kitsuyui.fileio"]
|
|
25
|
+
|
|
26
|
+
[tool.setuptools_scm]
|
|
27
|
+
version_file = "./src/kitsuyui/fileio/_version.py"
|
|
28
|
+
root = "../.."
|
|
29
|
+
|
|
30
|
+
[dependency-groups]
|
|
31
|
+
dev = ["dev-shared"]
|
|
32
|
+
|
|
33
|
+
[tool.uv.sources]
|
|
34
|
+
dev-shared = { workspace = true }
|
|
35
|
+
|
|
36
|
+
[tool.poe]
|
|
37
|
+
include = "../../dev-shared/poe.tasks.toml"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""File I/O primitives with explicit replacement and durability semantics."""
|
|
2
|
+
|
|
3
|
+
from ._atomic import (
|
|
4
|
+
Durability,
|
|
5
|
+
DurabilityError,
|
|
6
|
+
atomic_write,
|
|
7
|
+
atomic_write_bytes,
|
|
8
|
+
atomic_write_text,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"Durability",
|
|
13
|
+
"DurabilityError",
|
|
14
|
+
"atomic_write",
|
|
15
|
+
"atomic_write_bytes",
|
|
16
|
+
"atomic_write_text",
|
|
17
|
+
]
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""Atomically replace a single file with staged bytes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import contextlib
|
|
6
|
+
import os
|
|
7
|
+
import tempfile
|
|
8
|
+
from collections.abc import Generator
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import BinaryIO, Literal
|
|
12
|
+
|
|
13
|
+
Durability = Literal["none", "file", "file_and_directory"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class DurabilityError(OSError):
|
|
17
|
+
"""The replacement happened, but the directory could not be synced."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _validate_options(durability: Durability) -> None:
|
|
21
|
+
if durability not in ("none", "file", "file_and_directory"):
|
|
22
|
+
raise ValueError(f"invalid durability: {durability!r}")
|
|
23
|
+
if durability == "file_and_directory" and os.name != "posix":
|
|
24
|
+
raise NotImplementedError("directory sync is only supported on POSIX")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _validate_permissions(permissions: int | None) -> None:
|
|
28
|
+
if permissions is not None and not 0 <= permissions <= 0o777:
|
|
29
|
+
raise ValueError("permissions must be between 0 and 0o777")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _destination(path: str | os.PathLike[str]) -> Path:
|
|
33
|
+
given_path = Path(path)
|
|
34
|
+
if given_path.name in ("", ".", ".."):
|
|
35
|
+
raise ValueError("path must name a file")
|
|
36
|
+
# Resolve the parent only: the destination symlink must be replaced.
|
|
37
|
+
return given_path.parent.resolve() / given_path.name
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@contextlib.contextmanager
|
|
41
|
+
def _directory_handle(
|
|
42
|
+
destination: Path, durability: Durability
|
|
43
|
+
) -> Generator[int | None, None, None]:
|
|
44
|
+
fd = None
|
|
45
|
+
try:
|
|
46
|
+
if durability == "file_and_directory":
|
|
47
|
+
fd = os.open(
|
|
48
|
+
destination.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
|
|
49
|
+
)
|
|
50
|
+
yield fd
|
|
51
|
+
finally:
|
|
52
|
+
if fd is not None:
|
|
53
|
+
with contextlib.suppress(OSError):
|
|
54
|
+
os.close(fd)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class _StagedPath:
|
|
59
|
+
path: Path
|
|
60
|
+
owned: bool = True
|
|
61
|
+
|
|
62
|
+
def replace(self, destination: Path) -> None:
|
|
63
|
+
self.path.replace(destination)
|
|
64
|
+
self.owned = False
|
|
65
|
+
|
|
66
|
+
def cleanup(self) -> None:
|
|
67
|
+
if self.owned:
|
|
68
|
+
with contextlib.suppress(OSError):
|
|
69
|
+
self.path.unlink()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@contextlib.contextmanager
|
|
73
|
+
def _staged_file(
|
|
74
|
+
destination: Path,
|
|
75
|
+
) -> Generator[tuple[_StagedPath, BinaryIO], None, None]:
|
|
76
|
+
fd, name = tempfile.mkstemp(
|
|
77
|
+
prefix=".fileio-", suffix=".tmp", dir=destination.parent
|
|
78
|
+
)
|
|
79
|
+
temporary = _StagedPath(Path(name))
|
|
80
|
+
try:
|
|
81
|
+
with _owned_stream(fd) as stream:
|
|
82
|
+
yield temporary, stream
|
|
83
|
+
finally:
|
|
84
|
+
temporary.cleanup()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@contextlib.contextmanager
|
|
88
|
+
def _owned_stream(fd: int) -> Generator[BinaryIO, None, None]:
|
|
89
|
+
stream = _open_stream(fd)
|
|
90
|
+
try:
|
|
91
|
+
yield stream
|
|
92
|
+
except BaseException:
|
|
93
|
+
with contextlib.suppress(BaseException):
|
|
94
|
+
stream.close()
|
|
95
|
+
raise
|
|
96
|
+
else:
|
|
97
|
+
_close_stream(stream)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _close_stream(stream: BinaryIO) -> None:
|
|
101
|
+
if not stream.closed:
|
|
102
|
+
stream.close()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _open_stream(fd: int) -> BinaryIO:
|
|
106
|
+
try:
|
|
107
|
+
return os.fdopen(fd, "wb")
|
|
108
|
+
except BaseException:
|
|
109
|
+
with contextlib.suppress(OSError):
|
|
110
|
+
os.close(fd)
|
|
111
|
+
raise
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _prepare_stream(
|
|
115
|
+
stream: BinaryIO,
|
|
116
|
+
temporary: Path,
|
|
117
|
+
durability: Durability,
|
|
118
|
+
permissions: int | None,
|
|
119
|
+
) -> None:
|
|
120
|
+
_flush_stream(stream)
|
|
121
|
+
if permissions is not None:
|
|
122
|
+
_set_permissions(stream, temporary, permissions)
|
|
123
|
+
if durability != "none":
|
|
124
|
+
os.fsync(stream.fileno())
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _flush_stream(stream: BinaryIO) -> None:
|
|
128
|
+
if stream.closed:
|
|
129
|
+
raise ValueError("the atomic_write stream was closed by the caller")
|
|
130
|
+
stream.flush()
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _set_permissions(
|
|
134
|
+
stream: BinaryIO, temporary: Path, permissions: int
|
|
135
|
+
) -> None:
|
|
136
|
+
if hasattr(os, "fchmod"):
|
|
137
|
+
os.fchmod(stream.fileno(), permissions)
|
|
138
|
+
else:
|
|
139
|
+
temporary.chmod(permissions)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _commit(
|
|
143
|
+
temporary: _StagedPath, destination: Path, directory_fd: int | None
|
|
144
|
+
) -> None:
|
|
145
|
+
temporary.replace(destination)
|
|
146
|
+
if directory_fd is not None:
|
|
147
|
+
try:
|
|
148
|
+
os.fsync(directory_fd)
|
|
149
|
+
except OSError as exc:
|
|
150
|
+
raise DurabilityError(
|
|
151
|
+
f"file was replaced, but directory sync failed: {destination}"
|
|
152
|
+
) from exc
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@contextlib.contextmanager
|
|
156
|
+
def atomic_write(
|
|
157
|
+
path: str | os.PathLike[str],
|
|
158
|
+
*,
|
|
159
|
+
durability: Durability = "file",
|
|
160
|
+
permissions: int | None = None,
|
|
161
|
+
) -> Generator[BinaryIO, None, None]:
|
|
162
|
+
"""Yield a binary stream and replace the destination on successful exit.
|
|
163
|
+
|
|
164
|
+
The destination directory must exist. Temporary files are private (0600 on
|
|
165
|
+
POSIX); replacement does not preserve the original file's metadata.
|
|
166
|
+
"""
|
|
167
|
+
_validate_options(durability)
|
|
168
|
+
_validate_permissions(permissions)
|
|
169
|
+
destination = _destination(path)
|
|
170
|
+
with (
|
|
171
|
+
_directory_handle(destination, durability) as directory_fd,
|
|
172
|
+
_staged_file(destination) as (temporary, stream),
|
|
173
|
+
):
|
|
174
|
+
yield stream
|
|
175
|
+
_prepare_stream(stream, temporary.path, durability, permissions)
|
|
176
|
+
stream.close()
|
|
177
|
+
_commit(temporary, destination, directory_fd)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def atomic_write_bytes(
|
|
181
|
+
path: str | os.PathLike[str],
|
|
182
|
+
data: bytes,
|
|
183
|
+
*,
|
|
184
|
+
durability: Durability = "file",
|
|
185
|
+
permissions: int | None = None,
|
|
186
|
+
) -> None:
|
|
187
|
+
"""Replace a file with bytes after the entire write succeeds."""
|
|
188
|
+
with atomic_write(
|
|
189
|
+
path, durability=durability, permissions=permissions
|
|
190
|
+
) as stream:
|
|
191
|
+
stream.write(data)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def atomic_write_text(
|
|
195
|
+
path: str | os.PathLike[str],
|
|
196
|
+
text: str,
|
|
197
|
+
*,
|
|
198
|
+
encoding: str = "utf-8",
|
|
199
|
+
errors: str = "strict",
|
|
200
|
+
durability: Durability = "file",
|
|
201
|
+
permissions: int | None = None,
|
|
202
|
+
) -> None:
|
|
203
|
+
"""Encode text and replace a file after the entire write succeeds."""
|
|
204
|
+
atomic_write_bytes(
|
|
205
|
+
path,
|
|
206
|
+
text.encode(encoding, errors),
|
|
207
|
+
durability=durability,
|
|
208
|
+
permissions=permissions,
|
|
209
|
+
)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.8.1'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 8, 1)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = 'g7c51344f1'
|
|
File without changes
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kitsuyui.fileio
|
|
3
|
+
Version: 0.8.1
|
|
4
|
+
Summary: Explicit and safe file writing utilities
|
|
5
|
+
Author-email: Yui KITSU <kitsuyui+github@kitsuyui.com>
|
|
6
|
+
License-Expression: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://github.com/kitsuyui/pypi-playground
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Dynamic: license-file
|
|
13
|
+
|
|
14
|
+
# kitsuyui.fileio
|
|
15
|
+
|
|
16
|
+
Small, explicit file I/O primitives. The first release provides atomic
|
|
17
|
+
replacement of a single file without runtime dependencies (Python 3.10+).
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
from kitsuyui.fileio import atomic_write, atomic_write_bytes, atomic_write_text
|
|
21
|
+
|
|
22
|
+
atomic_write_text("settings.txt", "ready\n")
|
|
23
|
+
atomic_write_bytes("data.bin", b"\x00\x01")
|
|
24
|
+
with atomic_write("large.bin") as output:
|
|
25
|
+
output.write(b"first chunk")
|
|
26
|
+
output.write(b"second chunk")
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
`atomic_write` yields a binary stream. On normal exit it flushes and closes the
|
|
30
|
+
stream before replacing the destination. An exception inside the block leaves
|
|
31
|
+
the destination untouched and removes the temporary file when possible. Do not
|
|
32
|
+
close the yielded stream manually. The convenience functions accept paths and
|
|
33
|
+
bytes or text respectively; text is encoded as UTF-8 unless specified otherwise.
|
|
34
|
+
The parent directory must already exist. A caller that wraps the binary stream
|
|
35
|
+
in a `TextIOWrapper` must flush and detach the wrapper before leaving the block;
|
|
36
|
+
closing the supplied stream causes a `ValueError` and cancels replacement.
|
|
37
|
+
Text convenience writes use `str.encode` and do not translate newlines; callers
|
|
38
|
+
may choose the encoding and encoding error handler.
|
|
39
|
+
|
|
40
|
+
## Guarantees and boundaries
|
|
41
|
+
|
|
42
|
+
The temporary file is created in the destination directory and installed with
|
|
43
|
+
`os.replace`. On a filesystem supporting POSIX atomic rename, a reader opening
|
|
44
|
+
the destination sees either the previous file or the complete new file during
|
|
45
|
+
normal execution; an
|
|
46
|
+
already-open descriptor may continue to refer to the old file. On other systems
|
|
47
|
+
and filesystems, replacement guarantees depend on the platform. This does not
|
|
48
|
+
lock writers, prevent lost updates, make multiple replacements transactional, or
|
|
49
|
+
secure a directory controlled by an untrusted party. Concurrent successful
|
|
50
|
+
writers are last-replacement-wins. A destination symlink itself is replaced;
|
|
51
|
+
its target is not modified. The temporary file is created with restrictive
|
|
52
|
+
permissions (0600 on POSIX). Existing ownership, permissions, timestamps, ACLs,
|
|
53
|
+
extended attributes and hard links are not preserved. The restrictive default
|
|
54
|
+
avoids silently exposing content from a private file or temporary write. To
|
|
55
|
+
choose another POSIX permission mode, pass `permissions=0o644` (for example);
|
|
56
|
+
the specified bits are applied to the temporary file before replacement and
|
|
57
|
+
are not filtered by umask. Do not use permissive modes for sensitive data. On
|
|
58
|
+
Windows `os.chmod` has limited permission semantics. Copying existing permissions
|
|
59
|
+
is deliberately left to the caller because metadata and access policy are
|
|
60
|
+
platform-specific and can change concurrently.
|
|
61
|
+
|
|
62
|
+
The keyword `durability` distinguishes visibility from persistence:
|
|
63
|
+
|
|
64
|
+
* `"none"`: flush and close before replacement; no `fsync`.
|
|
65
|
+
* `"file"` (default): also `fsync` the staged file before replacement.
|
|
66
|
+
* `"file_and_directory"`: also `fsync` the parent directory after replacement.
|
|
67
|
+
This option is available only on POSIX and can still fail on a filesystem
|
|
68
|
+
that does not support directory sync.
|
|
69
|
+
|
|
70
|
+
Sync calls request persistence from the operating system; they cannot guarantee
|
|
71
|
+
the behavior of every filesystem, storage device, or hardware buffer. A failure
|
|
72
|
+
before `os.replace` leaves an existing destination unchanged. If directory
|
|
73
|
+
sync fails after replacement, `DurabilityError` is raised: the new destination
|
|
74
|
+
is already visible, but its persistence is uncertain. If replacement itself fails,
|
|
75
|
+
the operating system error is propagated. Abrupt process termination can leave a
|
|
76
|
+
temporary file behind; ordinary exception cleanup is best effort. A crash with
|
|
77
|
+
`"none"` can leave the new file empty or truncated even after the rename, so
|
|
78
|
+
the runtime visibility guarantee is not a crash-recovery guarantee. On macOS,
|
|
79
|
+
`os.fsync` does not request the stronger `F_FULLFSYNC` operation, so even a
|
|
80
|
+
successful sync does not guarantee that hardware buffers reached durable media.
|
|
81
|
+
|
|
82
|
+
## License
|
|
83
|
+
|
|
84
|
+
BSD 3-Clause License. See the repository's LICENSE.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/kitsuyui.fileio.egg-info/PKG-INFO
|
|
5
|
+
src/kitsuyui.fileio.egg-info/SOURCES.txt
|
|
6
|
+
src/kitsuyui.fileio.egg-info/dependency_links.txt
|
|
7
|
+
src/kitsuyui.fileio.egg-info/scm_file_list.json
|
|
8
|
+
src/kitsuyui.fileio.egg-info/scm_version.json
|
|
9
|
+
src/kitsuyui.fileio.egg-info/top_level.txt
|
|
10
|
+
src/kitsuyui/fileio/__init__.py
|
|
11
|
+
src/kitsuyui/fileio/_atomic.py
|
|
12
|
+
src/kitsuyui/fileio/_version.py
|
|
13
|
+
src/kitsuyui/fileio/py.typed
|
|
14
|
+
tests/test_atomic.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
kitsuyui
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import threading
|
|
5
|
+
from collections.abc import Generator
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, cast
|
|
8
|
+
from unittest.mock import patch
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
|
|
12
|
+
from kitsuyui.fileio import (
|
|
13
|
+
DurabilityError,
|
|
14
|
+
atomic_write,
|
|
15
|
+
atomic_write_bytes,
|
|
16
|
+
atomic_write_text,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_text_and_bytes_replace_existing_file(tmp_path: Path) -> None:
|
|
21
|
+
path = tmp_path / "data"
|
|
22
|
+
path.write_bytes(b"old")
|
|
23
|
+
atomic_write_text(path, "na\u00efve", encoding="utf-8")
|
|
24
|
+
assert path.read_bytes() == "na\u00efve".encode()
|
|
25
|
+
atomic_write_bytes(path, b"new")
|
|
26
|
+
assert path.read_bytes() == b"new"
|
|
27
|
+
assert list(tmp_path.iterdir()) == [path]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_context_error_preserves_original_and_removes_temporary(
|
|
31
|
+
tmp_path: Path,
|
|
32
|
+
) -> None:
|
|
33
|
+
path = tmp_path / "data"
|
|
34
|
+
path.write_bytes(b"old")
|
|
35
|
+
with pytest.raises(ValueError, match="stop"), atomic_write(path) as stream:
|
|
36
|
+
stream.write(b"partial")
|
|
37
|
+
raise ValueError("stop")
|
|
38
|
+
assert path.read_bytes() == b"old"
|
|
39
|
+
assert list(tmp_path.iterdir()) == [path]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@pytest.mark.parametrize(
|
|
43
|
+
("target", "operation"), [(Path, "replace"), (os, "fsync")]
|
|
44
|
+
)
|
|
45
|
+
def test_precommit_failure_keeps_original(
|
|
46
|
+
tmp_path: Path, target: Any, operation: str
|
|
47
|
+
) -> None:
|
|
48
|
+
path = tmp_path / "data"
|
|
49
|
+
path.write_bytes(b"old")
|
|
50
|
+
with (
|
|
51
|
+
patch.object(target, operation, side_effect=OSError("injected")),
|
|
52
|
+
pytest.raises(OSError, match="injected"),
|
|
53
|
+
):
|
|
54
|
+
atomic_write_bytes(path, b"new")
|
|
55
|
+
assert path.read_bytes() == b"old"
|
|
56
|
+
assert list(tmp_path.iterdir()) == [path]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@pytest.mark.skipif(os.name != "posix", reason="directory fsync is POSIX-only")
|
|
60
|
+
def test_directory_sync_failure_reports_visible_replacement(
|
|
61
|
+
tmp_path: Path,
|
|
62
|
+
) -> None:
|
|
63
|
+
path = tmp_path / "data"
|
|
64
|
+
path.write_bytes(b"old")
|
|
65
|
+
original_fsync = os.fsync
|
|
66
|
+
calls = 0
|
|
67
|
+
|
|
68
|
+
def fail_second(fd: int) -> None:
|
|
69
|
+
nonlocal calls
|
|
70
|
+
calls += 1
|
|
71
|
+
if calls == 2:
|
|
72
|
+
raise OSError("directory sync failed")
|
|
73
|
+
original_fsync(fd)
|
|
74
|
+
|
|
75
|
+
with (
|
|
76
|
+
patch.object(os, "fsync", side_effect=fail_second),
|
|
77
|
+
pytest.raises(DurabilityError),
|
|
78
|
+
):
|
|
79
|
+
atomic_write_bytes(path, b"new", durability="file_and_directory")
|
|
80
|
+
assert path.read_bytes() == b"new"
|
|
81
|
+
assert list(tmp_path.iterdir()) == [path]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _write_versions(path: Path, versions: tuple[bytes, bytes]) -> None:
|
|
85
|
+
for index in range(30):
|
|
86
|
+
atomic_write_bytes(path, versions[index % 2], durability="none")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@pytest.mark.skipif(
|
|
90
|
+
os.name == "nt", reason="Windows may deny replacement of an open file"
|
|
91
|
+
)
|
|
92
|
+
def test_reader_sees_only_complete_versions(tmp_path: Path) -> None:
|
|
93
|
+
path = tmp_path / "data"
|
|
94
|
+
versions = (b"a" * 32768, b"b" * 32768)
|
|
95
|
+
path.write_bytes(versions[0])
|
|
96
|
+
observed: list[bytes] = []
|
|
97
|
+
done = threading.Event()
|
|
98
|
+
|
|
99
|
+
def reader() -> None:
|
|
100
|
+
while not done.is_set():
|
|
101
|
+
observed.append(path.read_bytes())
|
|
102
|
+
|
|
103
|
+
thread = threading.Thread(target=reader)
|
|
104
|
+
thread.start()
|
|
105
|
+
try:
|
|
106
|
+
_write_versions(path, versions)
|
|
107
|
+
finally:
|
|
108
|
+
done.set()
|
|
109
|
+
thread.join()
|
|
110
|
+
assert observed
|
|
111
|
+
assert all(value in versions for value in observed)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@pytest.mark.skipif(os.name != "posix", reason="POSIX permission semantics")
|
|
115
|
+
def test_permissions_and_symlink_replacement(tmp_path: Path) -> None:
|
|
116
|
+
target = tmp_path / "target"
|
|
117
|
+
target.write_bytes(b"unchanged")
|
|
118
|
+
path = tmp_path / "link"
|
|
119
|
+
path.symlink_to(target)
|
|
120
|
+
atomic_write_bytes(path, b"new")
|
|
121
|
+
assert not path.is_symlink()
|
|
122
|
+
assert target.read_bytes() == b"unchanged"
|
|
123
|
+
assert path.stat().st_mode & 0o777 == 0o600
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_invalid_durability_does_not_create_files(tmp_path: Path) -> None:
|
|
127
|
+
with pytest.raises(ValueError):
|
|
128
|
+
atomic_write_bytes(
|
|
129
|
+
tmp_path / "data", b"new", durability=cast(Any, "unknown")
|
|
130
|
+
)
|
|
131
|
+
assert list(tmp_path.iterdir()) == []
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_body_exception_survives_close_failure(
|
|
135
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
136
|
+
) -> None:
|
|
137
|
+
path = tmp_path / "data"
|
|
138
|
+
path.write_bytes(b"old")
|
|
139
|
+
with (
|
|
140
|
+
pytest.raises(RuntimeError, match="body"),
|
|
141
|
+
atomic_write(path) as stream,
|
|
142
|
+
):
|
|
143
|
+
real_close = stream.close
|
|
144
|
+
|
|
145
|
+
def fail_close() -> None:
|
|
146
|
+
real_close()
|
|
147
|
+
raise OSError("close")
|
|
148
|
+
|
|
149
|
+
monkeypatch.setattr(stream, "close", fail_close)
|
|
150
|
+
raise RuntimeError("body")
|
|
151
|
+
assert path.read_bytes() == b"old"
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def test_caller_close_has_clear_error_and_preserves_old(
|
|
155
|
+
tmp_path: Path,
|
|
156
|
+
) -> None:
|
|
157
|
+
path = tmp_path / "data"
|
|
158
|
+
path.write_bytes(b"old")
|
|
159
|
+
with (
|
|
160
|
+
pytest.raises(ValueError, match="closed by the caller"),
|
|
161
|
+
atomic_write(path) as stream,
|
|
162
|
+
):
|
|
163
|
+
stream.write(b"new")
|
|
164
|
+
stream.close()
|
|
165
|
+
assert path.read_bytes() == b"old"
|
|
166
|
+
assert list(tmp_path.iterdir()) == [path]
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@pytest.mark.skipif(os.name != "posix", reason="POSIX permission semantics")
|
|
170
|
+
def test_explicit_permissions(tmp_path: Path) -> None:
|
|
171
|
+
path = tmp_path / "data"
|
|
172
|
+
atomic_write_text(path, "hi", permissions=0o640)
|
|
173
|
+
assert path.stat().st_mode & 0o777 == 0o640
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@pytest.mark.skipif(os.name != "posix", reason="symlink availability")
|
|
177
|
+
def test_parent_symlink_dotdot_resolves_like_os(tmp_path: Path) -> None:
|
|
178
|
+
nested = tmp_path / "other" / "nested"
|
|
179
|
+
nested.mkdir(parents=True)
|
|
180
|
+
(tmp_path / "link").symlink_to(nested, target_is_directory=True)
|
|
181
|
+
atomic_write_bytes(tmp_path / "link" / ".." / "data", b"yes")
|
|
182
|
+
assert (tmp_path / "other" / "data").read_bytes() == b"yes"
|
|
183
|
+
assert not (tmp_path / "data").exists()
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_long_destination_and_encoding_errors(tmp_path: Path) -> None:
|
|
187
|
+
path = tmp_path / ("x" * 240)
|
|
188
|
+
atomic_write_text(path, "na\u00efve", encoding="ascii", errors="replace")
|
|
189
|
+
assert path.read_bytes() == b"na?ve"
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def test_invalid_permissions_does_not_create_file(tmp_path: Path) -> None:
|
|
193
|
+
with pytest.raises(ValueError, match="permissions"):
|
|
194
|
+
atomic_write_bytes(tmp_path / "data", b"new", permissions=0o1000)
|
|
195
|
+
assert list(tmp_path.iterdir()) == []
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
@pytest.fixture
|
|
199
|
+
def reused_temporary_name() -> Generator[list[Path], None, None]:
|
|
200
|
+
recreated: list[Path] = []
|
|
201
|
+
real_replace = Path.replace
|
|
202
|
+
|
|
203
|
+
def replace_and_recreate(source: Path, destination: Path) -> Path:
|
|
204
|
+
result = real_replace(source, destination)
|
|
205
|
+
source.write_bytes(b"another writer")
|
|
206
|
+
recreated.append(source)
|
|
207
|
+
return result
|
|
208
|
+
|
|
209
|
+
with patch.object(Path, "replace", replace_and_recreate):
|
|
210
|
+
yield recreated
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def test_reused_temporary_name_survives_success(
|
|
214
|
+
tmp_path: Path, reused_temporary_name: list[Path]
|
|
215
|
+
) -> None:
|
|
216
|
+
path = tmp_path / "data"
|
|
217
|
+
atomic_write_bytes(path, b"new")
|
|
218
|
+
assert path.read_bytes() == b"new"
|
|
219
|
+
assert len(reused_temporary_name) == 1
|
|
220
|
+
assert reused_temporary_name[0].read_bytes() == b"another writer"
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@pytest.mark.skipif(os.name != "posix", reason="directory fsync is POSIX-only")
|
|
224
|
+
def test_reused_temporary_name_survives_directory_sync_failure(
|
|
225
|
+
tmp_path: Path, reused_temporary_name: list[Path]
|
|
226
|
+
) -> None:
|
|
227
|
+
path = tmp_path / "data"
|
|
228
|
+
with (
|
|
229
|
+
patch.object(os, "fsync", side_effect=[None, OSError("sync failed")]),
|
|
230
|
+
pytest.raises(DurabilityError),
|
|
231
|
+
):
|
|
232
|
+
atomic_write_bytes(path, b"new", durability="file_and_directory")
|
|
233
|
+
assert path.read_bytes() == b"new"
|
|
234
|
+
assert len(reused_temporary_name) == 1
|
|
235
|
+
assert reused_temporary_name[0].read_bytes() == b"another writer"
|