servatus 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- servatus/__init__.py +26 -0
- servatus/_errors.py +30 -0
- servatus/_posix.py +316 -0
- servatus/_workspace.py +306 -0
- servatus/py.typed +1 -0
- servatus-0.0.1.dist-info/METADATA +99 -0
- servatus-0.0.1.dist-info/RECORD +9 -0
- servatus-0.0.1.dist-info/WHEEL +4 -0
- servatus-0.0.1.dist-info/licenses/LICENSE +21 -0
servatus/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from ._errors import (
|
|
2
|
+
CrossDevicePublication,
|
|
3
|
+
DestinationExists,
|
|
4
|
+
PublicationError,
|
|
5
|
+
ServatusError,
|
|
6
|
+
UnsafePublication,
|
|
7
|
+
UnsupportedPlatform,
|
|
8
|
+
WorkConflict,
|
|
9
|
+
WorkspaceBusy,
|
|
10
|
+
)
|
|
11
|
+
from ._workspace import Draft, Publication, Workspace, publish
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"CrossDevicePublication",
|
|
15
|
+
"DestinationExists",
|
|
16
|
+
"Draft",
|
|
17
|
+
"Publication",
|
|
18
|
+
"PublicationError",
|
|
19
|
+
"ServatusError",
|
|
20
|
+
"UnsafePublication",
|
|
21
|
+
"UnsupportedPlatform",
|
|
22
|
+
"WorkConflict",
|
|
23
|
+
"Workspace",
|
|
24
|
+
"WorkspaceBusy",
|
|
25
|
+
"publish",
|
|
26
|
+
]
|
servatus/_errors.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
class ServatusError(Exception):
|
|
2
|
+
"""Base class for Servatus contract failures."""
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class PublicationError(ServatusError):
|
|
6
|
+
"""A draft could not be safely published."""
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class DestinationExists(PublicationError):
|
|
10
|
+
"""The requested destination or draft entry already exists."""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class CrossDevicePublication(PublicationError):
|
|
14
|
+
"""Publication would cross a filesystem boundary."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class UnsafePublication(PublicationError):
|
|
18
|
+
"""A path or filesystem entry violates the publication contract."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class UnsupportedPlatform(PublicationError):
|
|
22
|
+
"""The platform cannot provide an atomic no-replace commit."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class WorkConflict(ServatusError):
|
|
26
|
+
"""Existing private work is bound to another identity."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class WorkspaceBusy(ServatusError):
|
|
30
|
+
"""Another writer currently owns the workspace."""
|
servatus/_posix.py
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ctypes
|
|
4
|
+
import errno
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import stat
|
|
8
|
+
import sys
|
|
9
|
+
from collections.abc import Callable
|
|
10
|
+
from contextlib import suppress
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from ._errors import (
|
|
14
|
+
CrossDevicePublication,
|
|
15
|
+
DestinationExists,
|
|
16
|
+
PublicationError,
|
|
17
|
+
UnsafePublication,
|
|
18
|
+
UnsupportedPlatform,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
_RENAME_NOREPLACE = 1
|
|
22
|
+
_RENAME_EXCL = 0x00000004
|
|
23
|
+
_DIRECTORY_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW
|
|
24
|
+
_FILE_FLAGS = os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW | os.O_NONBLOCK
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def require_supported_platform() -> None:
|
|
28
|
+
if not (sys.platform.startswith("linux") or sys.platform == "darwin"):
|
|
29
|
+
raise UnsupportedPlatform(f"unsupported platform: {sys.platform}")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def normalize_destination(destination: Path) -> tuple[Path, str]:
|
|
33
|
+
raw = Path(destination)
|
|
34
|
+
reject_nul_path(raw)
|
|
35
|
+
if not raw.name or raw.name in {".", ".."}:
|
|
36
|
+
raise UnsafePublication("destination must name a directory inside an existing parent")
|
|
37
|
+
try:
|
|
38
|
+
parent = raw.parent.resolve(strict=True)
|
|
39
|
+
except (OSError, ValueError) as error:
|
|
40
|
+
raise UnsafePublication("destination parent does not exist") from error
|
|
41
|
+
if not parent.is_dir():
|
|
42
|
+
raise UnsafePublication("destination parent is not a directory")
|
|
43
|
+
return parent, raw.name
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def open_directory(path: Path) -> int:
|
|
47
|
+
reject_nul_path(path)
|
|
48
|
+
try:
|
|
49
|
+
return os.open(path, _DIRECTORY_FLAGS)
|
|
50
|
+
except OSError as error:
|
|
51
|
+
raise UnsafePublication(f"unsafe directory: {path}") from error
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def open_directory_at(parent_fd: int, name: str) -> int:
|
|
55
|
+
validate_leaf(name)
|
|
56
|
+
try:
|
|
57
|
+
descriptor = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent_fd)
|
|
58
|
+
except OSError as error:
|
|
59
|
+
raise UnsafePublication(f"unsafe directory entry: {name}") from error
|
|
60
|
+
parent_device = os.fstat(parent_fd).st_dev
|
|
61
|
+
if os.fstat(descriptor).st_dev != parent_device:
|
|
62
|
+
os.close(descriptor)
|
|
63
|
+
raise CrossDevicePublication(f"directory crosses a filesystem boundary: {name}")
|
|
64
|
+
return descriptor
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def same_entry(left: os.stat_result, right: os.stat_result) -> bool:
|
|
68
|
+
return left.st_dev == right.st_dev and left.st_ino == right.st_ino
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def ensure_entry(parent_fd: int, name: str, expected: os.stat_result) -> None:
|
|
72
|
+
validate_leaf(name)
|
|
73
|
+
try:
|
|
74
|
+
current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
|
75
|
+
except OSError as error:
|
|
76
|
+
raise UnsafePublication(f"filesystem entry disappeared: {name}") from error
|
|
77
|
+
if not same_entry(current, expected):
|
|
78
|
+
raise UnsafePublication(f"filesystem entry changed during publication: {name}")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def ensure_directory_path(path: Path, descriptor: int) -> None:
|
|
82
|
+
reject_nul_path(path)
|
|
83
|
+
try:
|
|
84
|
+
current = path.stat(follow_symlinks=False)
|
|
85
|
+
except OSError as error:
|
|
86
|
+
raise UnsafePublication(f"publication parent disappeared: {path}") from error
|
|
87
|
+
if not stat.S_ISDIR(current.st_mode) or not same_entry(current, os.fstat(descriptor)):
|
|
88
|
+
raise UnsafePublication(f"publication parent changed: {path}")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def make_directory_at(parent_fd: int, name: str) -> tuple[int, os.stat_result]:
|
|
92
|
+
validate_leaf(name)
|
|
93
|
+
with suppress(FileExistsError):
|
|
94
|
+
os.mkdir(name, mode=0o700, dir_fd=parent_fd)
|
|
95
|
+
descriptor = open_directory_at(parent_fd, name)
|
|
96
|
+
return descriptor, os.fstat(descriptor)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def make_unique_stage(parent_fd: int, prefix: str) -> tuple[str, int, os.stat_result]:
|
|
100
|
+
validate_leaf(prefix)
|
|
101
|
+
for _ in range(128):
|
|
102
|
+
name = f"{prefix}{os.urandom(12).hex()}"
|
|
103
|
+
try:
|
|
104
|
+
os.mkdir(name, mode=0o700, dir_fd=parent_fd)
|
|
105
|
+
except FileExistsError:
|
|
106
|
+
continue
|
|
107
|
+
entry = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
|
108
|
+
descriptor = -1
|
|
109
|
+
try:
|
|
110
|
+
descriptor = open_directory_at(parent_fd, name)
|
|
111
|
+
entry = os.fstat(descriptor)
|
|
112
|
+
sync_descriptor(parent_fd)
|
|
113
|
+
except BaseException as error:
|
|
114
|
+
cleanup_after_failure(parent_fd, name, entry, error)
|
|
115
|
+
if descriptor >= 0:
|
|
116
|
+
os.close(descriptor)
|
|
117
|
+
raise
|
|
118
|
+
return name, descriptor, entry
|
|
119
|
+
raise PublicationError("could not allocate a unique publication stage")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def sync_descriptor(descriptor: int) -> None:
|
|
123
|
+
os.fsync(descriptor)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def sync_tree(descriptor: int, device: int) -> None:
|
|
127
|
+
for name in sorted(os.listdir(descriptor)):
|
|
128
|
+
try:
|
|
129
|
+
entry = os.stat(name, dir_fd=descriptor, follow_symlinks=False)
|
|
130
|
+
except OSError as error:
|
|
131
|
+
raise UnsafePublication(f"draft entry disappeared: {name}") from error
|
|
132
|
+
if entry.st_dev != device:
|
|
133
|
+
raise CrossDevicePublication(f"draft entry crosses a filesystem boundary: {name}")
|
|
134
|
+
if stat.S_ISREG(entry.st_mode):
|
|
135
|
+
_sync_regular_file(descriptor, name, entry)
|
|
136
|
+
elif stat.S_ISDIR(entry.st_mode):
|
|
137
|
+
child = open_directory_at(descriptor, name)
|
|
138
|
+
try:
|
|
139
|
+
sync_tree(child, device)
|
|
140
|
+
finally:
|
|
141
|
+
os.close(child)
|
|
142
|
+
ensure_entry(descriptor, name, entry)
|
|
143
|
+
else:
|
|
144
|
+
raise UnsafePublication(f"draft contains a symlink or special file: {name}")
|
|
145
|
+
sync_descriptor(descriptor)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _sync_regular_file(parent_fd: int, name: str, expected: os.stat_result) -> None:
|
|
149
|
+
try:
|
|
150
|
+
descriptor = os.open(name, _FILE_FLAGS, dir_fd=parent_fd)
|
|
151
|
+
except OSError as error:
|
|
152
|
+
raise UnsafePublication(f"unsafe regular file: {name}") from error
|
|
153
|
+
try:
|
|
154
|
+
opened = os.fstat(descriptor)
|
|
155
|
+
if not stat.S_ISREG(opened.st_mode) or not same_entry(opened, expected):
|
|
156
|
+
raise UnsafePublication(f"draft file changed during publication: {name}")
|
|
157
|
+
sync_descriptor(descriptor)
|
|
158
|
+
synced = os.fstat(descriptor)
|
|
159
|
+
current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
|
160
|
+
if not same_entry(synced, current):
|
|
161
|
+
raise UnsafePublication(f"draft file changed during publication: {name}")
|
|
162
|
+
if (opened.st_size, opened.st_mtime_ns) != (synced.st_size, synced.st_mtime_ns):
|
|
163
|
+
raise UnsafePublication(f"draft file was modified during publication: {name}")
|
|
164
|
+
finally:
|
|
165
|
+
os.close(descriptor)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def reject_nul_path(path: Path) -> None:
|
|
169
|
+
if "\0" in os.fspath(path):
|
|
170
|
+
raise UnsafePublication(f"path contains an embedded NUL: {path!r}")
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def validate_leaf(name: str) -> None:
|
|
174
|
+
if not name or name in {".", ".."} or "/" in name or "\0" in name:
|
|
175
|
+
raise UnsafePublication(f"unsafe filesystem leaf: {name!r}")
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def commit_noreplace(parent_fd: int, source: str, destination: str) -> None:
|
|
179
|
+
require_supported_platform()
|
|
180
|
+
validate_leaf(source)
|
|
181
|
+
validate_leaf(destination)
|
|
182
|
+
if sys.platform.startswith("linux"):
|
|
183
|
+
_linux_rename_noreplace(parent_fd, source, destination)
|
|
184
|
+
else:
|
|
185
|
+
_macos_rename_noreplace(parent_fd, source, destination)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _linux_rename_noreplace(parent_fd: int, source: str, destination: str) -> None:
|
|
189
|
+
library = ctypes.CDLL(None, use_errno=True)
|
|
190
|
+
try:
|
|
191
|
+
renameat2 = library.renameat2
|
|
192
|
+
except AttributeError as error:
|
|
193
|
+
raise UnsupportedPlatform("libc does not expose renameat2") from error
|
|
194
|
+
renameat2.argtypes = [
|
|
195
|
+
ctypes.c_int,
|
|
196
|
+
ctypes.c_char_p,
|
|
197
|
+
ctypes.c_int,
|
|
198
|
+
ctypes.c_char_p,
|
|
199
|
+
ctypes.c_uint,
|
|
200
|
+
]
|
|
201
|
+
renameat2.restype = ctypes.c_int
|
|
202
|
+
result = renameat2(
|
|
203
|
+
parent_fd,
|
|
204
|
+
os.fsencode(source),
|
|
205
|
+
parent_fd,
|
|
206
|
+
os.fsencode(destination),
|
|
207
|
+
_RENAME_NOREPLACE,
|
|
208
|
+
)
|
|
209
|
+
if result != 0:
|
|
210
|
+
_raise_rename_error(ctypes.get_errno())
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _macos_rename_noreplace(parent_fd: int, source: str, destination: str) -> None:
|
|
214
|
+
library = ctypes.CDLL(None, use_errno=True)
|
|
215
|
+
try:
|
|
216
|
+
renameatx_np = library.renameatx_np
|
|
217
|
+
except AttributeError as error:
|
|
218
|
+
raise UnsupportedPlatform("libc does not expose renameatx_np") from error
|
|
219
|
+
renameatx_np.argtypes = [
|
|
220
|
+
ctypes.c_int,
|
|
221
|
+
ctypes.c_char_p,
|
|
222
|
+
ctypes.c_int,
|
|
223
|
+
ctypes.c_char_p,
|
|
224
|
+
ctypes.c_uint,
|
|
225
|
+
]
|
|
226
|
+
renameatx_np.restype = ctypes.c_int
|
|
227
|
+
result = renameatx_np(
|
|
228
|
+
parent_fd,
|
|
229
|
+
os.fsencode(source),
|
|
230
|
+
parent_fd,
|
|
231
|
+
os.fsencode(destination),
|
|
232
|
+
_RENAME_EXCL,
|
|
233
|
+
)
|
|
234
|
+
if result != 0:
|
|
235
|
+
_raise_rename_error(ctypes.get_errno())
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _raise_rename_error(error_number: int) -> None:
|
|
239
|
+
if error_number in {errno.EEXIST, errno.ENOTEMPTY}:
|
|
240
|
+
raise DestinationExists("publication destination already exists")
|
|
241
|
+
if error_number == errno.EXDEV:
|
|
242
|
+
raise CrossDevicePublication("publication commit crosses a filesystem boundary")
|
|
243
|
+
if error_number in {errno.EINVAL, errno.ENOSYS, errno.EOPNOTSUPP}:
|
|
244
|
+
raise UnsupportedPlatform("filesystem does not support atomic no-replace rename")
|
|
245
|
+
raise OSError(error_number, os.strerror(error_number))
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def remove_tree_at(parent_fd: int, name: str, expected: os.stat_result) -> None:
|
|
249
|
+
ensure_entry(parent_fd, name, expected)
|
|
250
|
+
if not shutil.rmtree.avoids_symlink_attacks:
|
|
251
|
+
raise UnsupportedPlatform("safe directory removal is unavailable")
|
|
252
|
+
shutil.rmtree(name, dir_fd=parent_fd)
|
|
253
|
+
sync_descriptor(parent_fd)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def remove_file_at(parent_fd: int, name: str, expected: os.stat_result) -> None:
|
|
257
|
+
ensure_entry(parent_fd, name, expected)
|
|
258
|
+
if not stat.S_ISREG(expected.st_mode):
|
|
259
|
+
raise UnsafePublication(f"cleanup target is not a regular file: {name}")
|
|
260
|
+
os.unlink(name, dir_fd=parent_fd)
|
|
261
|
+
sync_descriptor(parent_fd)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def cleanup_after_failure(
|
|
265
|
+
parent_fd: int,
|
|
266
|
+
name: str,
|
|
267
|
+
expected: os.stat_result,
|
|
268
|
+
error: BaseException,
|
|
269
|
+
) -> None:
|
|
270
|
+
try:
|
|
271
|
+
remove_tree_at(parent_fd, name, expected)
|
|
272
|
+
except Exception as cleanup_error:
|
|
273
|
+
error.add_note(f"Servatus could not remove failed stage: {cleanup_error}")
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def publication_attempt(
|
|
277
|
+
destination: Path,
|
|
278
|
+
build: Callable[[Path, int, int], None],
|
|
279
|
+
) -> Path:
|
|
280
|
+
require_supported_platform()
|
|
281
|
+
parent, destination_name = normalize_destination(destination)
|
|
282
|
+
parent_fd = open_directory(parent)
|
|
283
|
+
try:
|
|
284
|
+
return publication_attempt_at(parent, parent_fd, destination_name, build)
|
|
285
|
+
finally:
|
|
286
|
+
os.close(parent_fd)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def publication_attempt_at(
|
|
290
|
+
parent: Path,
|
|
291
|
+
parent_fd: int,
|
|
292
|
+
destination_name: str,
|
|
293
|
+
build: Callable[[Path, int, int], None],
|
|
294
|
+
) -> Path:
|
|
295
|
+
ensure_directory_path(parent, parent_fd)
|
|
296
|
+
stage_name = ""
|
|
297
|
+
stage_fd = -1
|
|
298
|
+
stage_entry: os.stat_result | None = None
|
|
299
|
+
try:
|
|
300
|
+
stage_name, stage_fd, stage_entry = make_unique_stage(parent_fd, ".servatus-stage-")
|
|
301
|
+
build(parent / stage_name, stage_fd, stage_entry.st_dev)
|
|
302
|
+
ensure_entry(parent_fd, stage_name, stage_entry)
|
|
303
|
+
sync_tree(stage_fd, stage_entry.st_dev)
|
|
304
|
+
ensure_entry(parent_fd, stage_name, stage_entry)
|
|
305
|
+
ensure_directory_path(parent, parent_fd)
|
|
306
|
+
commit_noreplace(parent_fd, stage_name, destination_name)
|
|
307
|
+
sync_descriptor(parent_fd)
|
|
308
|
+
return parent / destination_name
|
|
309
|
+
except BaseException as error:
|
|
310
|
+
if stage_name and stage_entry is not None:
|
|
311
|
+
cleanup_after_failure(parent_fd, stage_name, stage_entry, error)
|
|
312
|
+
raise
|
|
313
|
+
finally:
|
|
314
|
+
if stage_fd >= 0:
|
|
315
|
+
with suppress(OSError):
|
|
316
|
+
os.close(stage_fd)
|
servatus/_workspace.py
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import errno
|
|
4
|
+
import fcntl
|
|
5
|
+
import hashlib
|
|
6
|
+
import os
|
|
7
|
+
import stat
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from contextlib import suppress
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import Path, PurePosixPath
|
|
12
|
+
|
|
13
|
+
from . import _posix
|
|
14
|
+
from ._errors import (
|
|
15
|
+
CrossDevicePublication,
|
|
16
|
+
DestinationExists,
|
|
17
|
+
UnsafePublication,
|
|
18
|
+
WorkConflict,
|
|
19
|
+
WorkspaceBusy,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True, slots=True)
|
|
24
|
+
class Publication:
|
|
25
|
+
destination: Path
|
|
26
|
+
cleanup_pending: bool
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Draft:
|
|
30
|
+
def __init__(self, path: Path, descriptor: int, device: int) -> None:
|
|
31
|
+
self._path = path
|
|
32
|
+
self._descriptor = descriptor
|
|
33
|
+
self._device = device
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def path(self) -> Path:
|
|
37
|
+
return self._path
|
|
38
|
+
|
|
39
|
+
def link(self, source: Path, destination: str | PurePosixPath) -> None:
|
|
40
|
+
components = _safe_components(destination)
|
|
41
|
+
source_path = Path(source)
|
|
42
|
+
_posix.reject_nul_path(source_path)
|
|
43
|
+
try:
|
|
44
|
+
source_entry = source_path.stat(follow_symlinks=False)
|
|
45
|
+
except OSError as error:
|
|
46
|
+
raise UnsafePublication(f"hard-link source is unavailable: {source_path}") from error
|
|
47
|
+
if not stat.S_ISREG(source_entry.st_mode):
|
|
48
|
+
raise UnsafePublication(f"hard-link source is not a regular file: {source_path}")
|
|
49
|
+
try:
|
|
50
|
+
source_fd = os.open(
|
|
51
|
+
source_path,
|
|
52
|
+
os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW | os.O_NONBLOCK,
|
|
53
|
+
)
|
|
54
|
+
except OSError as error:
|
|
55
|
+
raise UnsafePublication(f"unsafe hard-link source: {source_path}") from error
|
|
56
|
+
try:
|
|
57
|
+
opened_source = os.fstat(source_fd)
|
|
58
|
+
if not stat.S_ISREG(opened_source.st_mode) or not _posix.same_entry(
|
|
59
|
+
opened_source, source_entry
|
|
60
|
+
):
|
|
61
|
+
raise UnsafePublication(f"hard-link source changed: {source_path}")
|
|
62
|
+
if opened_source.st_dev != self._device:
|
|
63
|
+
raise CrossDevicePublication(
|
|
64
|
+
f"hard-link source is on another filesystem: {source_path}"
|
|
65
|
+
)
|
|
66
|
+
self._link_open_source(source_path, opened_source, components)
|
|
67
|
+
finally:
|
|
68
|
+
os.close(source_fd)
|
|
69
|
+
|
|
70
|
+
def _link_open_source(
|
|
71
|
+
self,
|
|
72
|
+
source: Path,
|
|
73
|
+
source_entry: os.stat_result,
|
|
74
|
+
components: tuple[str, ...],
|
|
75
|
+
) -> None:
|
|
76
|
+
current_fd = os.dup(self._descriptor)
|
|
77
|
+
try:
|
|
78
|
+
for component in components[:-1]:
|
|
79
|
+
with suppress(FileExistsError):
|
|
80
|
+
os.mkdir(component, mode=0o700, dir_fd=current_fd)
|
|
81
|
+
child_fd = _posix.open_directory_at(current_fd, component)
|
|
82
|
+
os.close(current_fd)
|
|
83
|
+
current_fd = child_fd
|
|
84
|
+
leaf = components[-1]
|
|
85
|
+
try:
|
|
86
|
+
os.link(source, leaf, dst_dir_fd=current_fd, follow_symlinks=False)
|
|
87
|
+
except FileExistsError as error:
|
|
88
|
+
raise DestinationExists(
|
|
89
|
+
f"draft path already exists: {PurePosixPath(*components)}"
|
|
90
|
+
) from error
|
|
91
|
+
except OSError as error:
|
|
92
|
+
if error.errno == errno.EXDEV:
|
|
93
|
+
raise CrossDevicePublication(
|
|
94
|
+
f"hard-link source is on another filesystem: {source}"
|
|
95
|
+
) from error
|
|
96
|
+
raise
|
|
97
|
+
linked = os.stat(leaf, dir_fd=current_fd, follow_symlinks=False)
|
|
98
|
+
if not stat.S_ISREG(linked.st_mode) or not _posix.same_entry(linked, source_entry):
|
|
99
|
+
with suppress(OSError):
|
|
100
|
+
os.unlink(leaf, dir_fd=current_fd)
|
|
101
|
+
raise UnsafePublication(f"hard-link source changed: {source}")
|
|
102
|
+
finally:
|
|
103
|
+
os.close(current_fd)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _safe_components(destination: str | PurePosixPath) -> tuple[str, ...]:
|
|
107
|
+
if "\0" in os.fspath(destination):
|
|
108
|
+
raise UnsafePublication(f"draft path contains an embedded NUL: {destination!r}")
|
|
109
|
+
path = PurePosixPath(destination)
|
|
110
|
+
if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts):
|
|
111
|
+
raise UnsafePublication(f"draft path must be safe and relative: {destination}")
|
|
112
|
+
return path.parts
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _build_draft(build: Callable[[Draft], None]) -> Callable[[Path, int, int], None]:
|
|
116
|
+
def run(path: Path, descriptor: int, device: int) -> None:
|
|
117
|
+
build(Draft(path, descriptor, device))
|
|
118
|
+
|
|
119
|
+
return run
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def publish(destination: Path, build: Callable[[Draft], None]) -> Publication:
|
|
123
|
+
committed = _posix.publication_attempt(destination, _build_draft(build))
|
|
124
|
+
return Publication(committed, cleanup_pending=False)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class Workspace:
|
|
128
|
+
def __init__(self, destination: Path, *, identity: bytes) -> None:
|
|
129
|
+
parent, destination_name = _posix.normalize_destination(destination)
|
|
130
|
+
destination_key = hashlib.sha256(os.fsencode(destination_name)).hexdigest()[:24]
|
|
131
|
+
self._destination = parent / destination_name
|
|
132
|
+
self._parent = parent
|
|
133
|
+
self._parent_fd = -1
|
|
134
|
+
self._container_fd = -1
|
|
135
|
+
self._work_fd = -1
|
|
136
|
+
self._lock_fd = -1
|
|
137
|
+
self._container_entry: os.stat_result | None = None
|
|
138
|
+
self._container_name = f".servatus-{destination_key}.work"
|
|
139
|
+
self._identity = hashlib.sha256(identity).hexdigest().encode("ascii") + b"\n"
|
|
140
|
+
self._entered = False
|
|
141
|
+
self._published = False
|
|
142
|
+
|
|
143
|
+
def __enter__(self) -> Workspace:
|
|
144
|
+
if self._entered:
|
|
145
|
+
raise RuntimeError("workspace is already entered")
|
|
146
|
+
_posix.require_supported_platform()
|
|
147
|
+
self._parent_fd = _posix.open_directory(self._parent)
|
|
148
|
+
try:
|
|
149
|
+
self._container_fd, self._container_entry = _posix.make_directory_at(
|
|
150
|
+
self._parent_fd, self._container_name
|
|
151
|
+
)
|
|
152
|
+
self._lock_fd = self._open_lock()
|
|
153
|
+
try:
|
|
154
|
+
fcntl.flock(self._lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
155
|
+
except BlockingIOError as error:
|
|
156
|
+
raise WorkspaceBusy(f"workspace is already locked: {self._destination}") from error
|
|
157
|
+
self._bind_identity()
|
|
158
|
+
self._work_fd, _ = _posix.make_directory_at(self._container_fd, "work")
|
|
159
|
+
_posix.sync_descriptor(self._container_fd)
|
|
160
|
+
except BaseException:
|
|
161
|
+
self._close()
|
|
162
|
+
raise
|
|
163
|
+
self._entered = True
|
|
164
|
+
return self
|
|
165
|
+
|
|
166
|
+
def __exit__(self, *_: object) -> None:
|
|
167
|
+
self._close()
|
|
168
|
+
|
|
169
|
+
@property
|
|
170
|
+
def path(self) -> Path:
|
|
171
|
+
return self._parent / self._container_name / "work"
|
|
172
|
+
|
|
173
|
+
def publish(self, build: Callable[[Draft], None]) -> Publication:
|
|
174
|
+
if not self._entered or self._parent_fd < 0 or self._container_entry is None:
|
|
175
|
+
raise RuntimeError("workspace must be entered before publication")
|
|
176
|
+
if self._published:
|
|
177
|
+
raise RuntimeError("workspace has already published")
|
|
178
|
+
_posix.ensure_entry(self._parent_fd, self._container_name, self._container_entry)
|
|
179
|
+
self._verify_identity()
|
|
180
|
+
committed = _posix.publication_attempt_at(
|
|
181
|
+
self._parent,
|
|
182
|
+
self._parent_fd,
|
|
183
|
+
self._destination.name,
|
|
184
|
+
_build_draft(build),
|
|
185
|
+
)
|
|
186
|
+
cleanup_pending = False
|
|
187
|
+
try:
|
|
188
|
+
_cleanup_workspace(
|
|
189
|
+
self._parent_fd,
|
|
190
|
+
self._container_name,
|
|
191
|
+
self._container_entry,
|
|
192
|
+
)
|
|
193
|
+
except Exception:
|
|
194
|
+
cleanup_pending = True
|
|
195
|
+
self._published = True
|
|
196
|
+
return Publication(committed, cleanup_pending=cleanup_pending)
|
|
197
|
+
|
|
198
|
+
def _open_lock(self) -> int:
|
|
199
|
+
assert self._container_fd >= 0
|
|
200
|
+
try:
|
|
201
|
+
descriptor = os.open(
|
|
202
|
+
".lock",
|
|
203
|
+
os.O_RDWR | os.O_CREAT | os.O_CLOEXEC | os.O_NOFOLLOW,
|
|
204
|
+
0o600,
|
|
205
|
+
dir_fd=self._container_fd,
|
|
206
|
+
)
|
|
207
|
+
except OSError as error:
|
|
208
|
+
raise UnsafePublication("workspace lock is unsafe") from error
|
|
209
|
+
lock_entry = os.fstat(descriptor)
|
|
210
|
+
if not stat.S_ISREG(lock_entry.st_mode):
|
|
211
|
+
os.close(descriptor)
|
|
212
|
+
raise UnsafePublication("workspace lock is not a regular file")
|
|
213
|
+
return descriptor
|
|
214
|
+
|
|
215
|
+
def _bind_identity(self) -> None:
|
|
216
|
+
assert self._container_fd >= 0
|
|
217
|
+
try:
|
|
218
|
+
os.stat(".identity", dir_fd=self._container_fd, follow_symlinks=False)
|
|
219
|
+
except FileNotFoundError:
|
|
220
|
+
self._initialize_identity()
|
|
221
|
+
else:
|
|
222
|
+
self._verify_identity()
|
|
223
|
+
|
|
224
|
+
def _initialize_identity(self) -> None:
|
|
225
|
+
assert self._container_fd >= 0
|
|
226
|
+
stage_name = f".identity-{os.urandom(12).hex()}.tmp"
|
|
227
|
+
descriptor = -1
|
|
228
|
+
stage_entry: os.stat_result | None = None
|
|
229
|
+
try:
|
|
230
|
+
descriptor = os.open(
|
|
231
|
+
stage_name,
|
|
232
|
+
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC | os.O_NOFOLLOW,
|
|
233
|
+
0o600,
|
|
234
|
+
dir_fd=self._container_fd,
|
|
235
|
+
)
|
|
236
|
+
stage_entry = os.fstat(descriptor)
|
|
237
|
+
view = memoryview(self._identity)
|
|
238
|
+
while view:
|
|
239
|
+
written = os.write(descriptor, view)
|
|
240
|
+
view = view[written:]
|
|
241
|
+
_posix.sync_descriptor(descriptor)
|
|
242
|
+
os.close(descriptor)
|
|
243
|
+
descriptor = -1
|
|
244
|
+
try:
|
|
245
|
+
_posix.commit_noreplace(self._container_fd, stage_name, ".identity")
|
|
246
|
+
except DestinationExists:
|
|
247
|
+
self._verify_identity()
|
|
248
|
+
_posix.remove_file_at(self._container_fd, stage_name, stage_entry)
|
|
249
|
+
stage_entry = None
|
|
250
|
+
_posix.sync_descriptor(self._container_fd)
|
|
251
|
+
except BaseException as error:
|
|
252
|
+
if stage_entry is not None:
|
|
253
|
+
try:
|
|
254
|
+
_posix.remove_file_at(self._container_fd, stage_name, stage_entry)
|
|
255
|
+
except Exception as cleanup_error:
|
|
256
|
+
error.add_note(f"Servatus could not remove identity stage: {cleanup_error}")
|
|
257
|
+
raise
|
|
258
|
+
finally:
|
|
259
|
+
if descriptor >= 0:
|
|
260
|
+
os.close(descriptor)
|
|
261
|
+
|
|
262
|
+
def _verify_identity(self) -> None:
|
|
263
|
+
assert self._container_fd >= 0
|
|
264
|
+
try:
|
|
265
|
+
descriptor = os.open(
|
|
266
|
+
".identity",
|
|
267
|
+
os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW | os.O_NONBLOCK,
|
|
268
|
+
dir_fd=self._container_fd,
|
|
269
|
+
)
|
|
270
|
+
except OSError as error:
|
|
271
|
+
raise WorkConflict(f"workspace identity is unavailable: {self._destination}") from error
|
|
272
|
+
try:
|
|
273
|
+
identity_entry = os.fstat(descriptor)
|
|
274
|
+
if (
|
|
275
|
+
not stat.S_ISREG(identity_entry.st_mode)
|
|
276
|
+
or identity_entry.st_dev != os.fstat(self._container_fd).st_dev
|
|
277
|
+
or identity_entry.st_size != len(self._identity)
|
|
278
|
+
):
|
|
279
|
+
raise WorkConflict(f"workspace identity is invalid: {self._destination}")
|
|
280
|
+
chunks: list[bytes] = []
|
|
281
|
+
remaining = len(self._identity) + 1
|
|
282
|
+
while remaining and (chunk := os.read(descriptor, remaining)):
|
|
283
|
+
chunks.append(chunk)
|
|
284
|
+
remaining -= len(chunk)
|
|
285
|
+
if b"".join(chunks) != self._identity:
|
|
286
|
+
raise WorkConflict(f"workspace belongs to different work: {self._destination}")
|
|
287
|
+
_posix.ensure_entry(self._container_fd, ".identity", identity_entry)
|
|
288
|
+
finally:
|
|
289
|
+
os.close(descriptor)
|
|
290
|
+
|
|
291
|
+
def _close(self) -> None:
|
|
292
|
+
for attribute in ("_work_fd", "_lock_fd", "_container_fd", "_parent_fd"):
|
|
293
|
+
descriptor = getattr(self, attribute)
|
|
294
|
+
if descriptor >= 0:
|
|
295
|
+
with suppress(OSError):
|
|
296
|
+
os.close(descriptor)
|
|
297
|
+
setattr(self, attribute, -1)
|
|
298
|
+
self._entered = False
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _cleanup_workspace(
|
|
302
|
+
parent_fd: int,
|
|
303
|
+
name: str,
|
|
304
|
+
expected: os.stat_result,
|
|
305
|
+
) -> None:
|
|
306
|
+
_posix.remove_tree_at(parent_fd, name, expected)
|
servatus/py.typed
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: servatus
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Run resumable work through Slurm and atomically publish validated outputs.
|
|
5
|
+
Project-URL: Repository, https://github.com/edoski/servatus
|
|
6
|
+
Project-URL: Issues, https://github.com/edoski/servatus/issues
|
|
7
|
+
Author: Edoardo Galli
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: MacOS
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# Servatus
|
|
23
|
+
|
|
24
|
+
Run resumable work through Slurm and atomically publish validated outputs.
|
|
25
|
+
|
|
26
|
+
Servatus 0.0.1 is a publication-only alpha. It provides a small Python interface for keeping
|
|
27
|
+
private resumable work and exposing an application-built directory atomically. Slurm campaigns
|
|
28
|
+
arrive in a later release.
|
|
29
|
+
|
|
30
|
+
```sh
|
|
31
|
+
pip install servatus
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Publication
|
|
35
|
+
|
|
36
|
+
Use `publish` when failed work is disposable:
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
|
|
41
|
+
from servatus import Draft, publish
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def build(draft: Draft) -> None:
|
|
45
|
+
(draft.path / "result.json").write_text('{"status":"complete"}\n')
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
publication = publish(Path("outputs/run-1"), build)
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Use `Workspace` when a worker must retain private checkpoints across restarts:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from servatus import Draft, Workspace
|
|
55
|
+
|
|
56
|
+
destination = Path("outputs/model-1")
|
|
57
|
+
with Workspace(destination, identity=b"model request bytes") as workspace:
|
|
58
|
+
checkpoint = workspace.path / "last.ckpt"
|
|
59
|
+
# The application creates or resumes its own checkpoint here.
|
|
60
|
+
|
|
61
|
+
def assemble(draft: Draft) -> None:
|
|
62
|
+
draft.link(checkpoint, "last.ckpt")
|
|
63
|
+
# Perform application validation before returning.
|
|
64
|
+
|
|
65
|
+
publication = workspace.publish(assemble)
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`Workspace` binds its stable hidden state to the SHA-256 digest of the opaque identity and holds a
|
|
69
|
+
nonblocking writer lock. `Draft.link` only hard-links regular files into a safe relative path. The
|
|
70
|
+
application must not mutate a linked source inode after `Draft.link()` returns and before
|
|
71
|
+
publication completes. It owns file contents, validation, schemas, and completion meaning.
|
|
72
|
+
|
|
73
|
+
## Guarantees
|
|
74
|
+
|
|
75
|
+
- A destination is absent or one complete directory; it is never overwritten.
|
|
76
|
+
- Work, hard-link sources, stages, and destination must share a filesystem.
|
|
77
|
+
- Files and directories are synced before a kernel-exclusive commit; the destination parent is
|
|
78
|
+
synced afterward.
|
|
79
|
+
- Builder failures expose no destination. Resumable work remains available, while disposable
|
|
80
|
+
stages are removed.
|
|
81
|
+
- Successful workspace publication removes private state. A cleanup failure returns
|
|
82
|
+
`Publication(cleanup_pending=True)` without misreporting the committed destination as failed.
|
|
83
|
+
- Symlinks, special files, escaping link paths, path substitution, and unsupported exclusive-rename
|
|
84
|
+
primitives fail closed.
|
|
85
|
+
|
|
86
|
+
Servatus supports POSIX filesystems on Linux and macOS. Hardware durability still depends on the
|
|
87
|
+
filesystem and mount. It is not a security boundary against another process that can arbitrarily
|
|
88
|
+
modify the same parent directory.
|
|
89
|
+
|
|
90
|
+
## Non-goals in 0.0.1
|
|
91
|
+
|
|
92
|
+
Servatus does not interpret checkpoints or ML artifacts, decide when work is valid, model workflow
|
|
93
|
+
graphs, migrate old outputs, copy across filesystems, overwrite destinations, or provide scheduler
|
|
94
|
+
execution. It has no plugins, callbacks beyond the one build seam, runtime dependencies, daemon,
|
|
95
|
+
database, or global run store.
|
|
96
|
+
|
|
97
|
+
The public API is `Draft`, `Workspace`, `Publication`, `publish`, and the compact errors exported by
|
|
98
|
+
`servatus`. See [the context glossary](docs/CONTEXT.md) and [architecture decisions](docs/adr/README.md)
|
|
99
|
+
for the ownership boundary.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
servatus/__init__.py,sha256=AGI3N-Gg96BqAPHOtyy2DLabYzz3HcENi36cqCIjVss,537
|
|
2
|
+
servatus/_errors.py,sha256=Q4nndE_WoH9AKcnbRPsOB6NkO7INcCdCg-8XpFCZBmA,817
|
|
3
|
+
servatus/_posix.py,sha256=RnkBU-xSJEPAwvHKlZtxUE81kVNS_wICKA3oSob_I7w,11250
|
|
4
|
+
servatus/_workspace.py,sha256=fEskWfiZOMfXwqCaNsokeRcmu0y4CJReMQ9TArbqAKg,11820
|
|
5
|
+
servatus/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
6
|
+
servatus-0.0.1.dist-info/METADATA,sha256=LTQQC8d47hdds7wbYWuuzM_19VvoZobXxldPBr5aYDg,3890
|
|
7
|
+
servatus-0.0.1.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
8
|
+
servatus-0.0.1.dist-info/licenses/LICENSE,sha256=wV0fwxnUW3dIricT7A8U12uqgYn9qoXADUbb2P19U7w,1070
|
|
9
|
+
servatus-0.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Edoardo Galli
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|