polyctl 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyctl/__init__.py +7 -0
- polyctl/__main__.py +5 -0
- polyctl/a2a.py +226 -0
- polyctl/a2a_mcp.py +546 -0
- polyctl/acp.py +417 -0
- polyctl/adapters/README.md +101 -0
- polyctl/adapters/aider.json +32 -0
- polyctl/adapters/amazon-q.json +15 -0
- polyctl/adapters/amp.json +16 -0
- polyctl/adapters/antigravity.json +120 -0
- polyctl/adapters/claude-code.json +229 -0
- polyctl/adapters/cloud-generic.json +57 -0
- polyctl/adapters/codex.json +47 -0
- polyctl/adapters/command-code.json +45 -0
- polyctl/adapters/copilot.json +17 -0
- polyctl/adapters/cursor-agent.json +29 -0
- polyctl/adapters/deepseek.json +13 -0
- polyctl/adapters/droid.json +34 -0
- polyctl/adapters/gemini.json +30 -0
- polyctl/adapters/grok.json +22 -0
- polyctl/adapters/hermes.json +34 -0
- polyctl/adapters/kilocode.json +30 -0
- polyctl/adapters/kimi.json +23 -0
- polyctl/adapters/kiro.json +332 -0
- polyctl/adapters/ollama.json +29 -0
- polyctl/adapters/omp.json +34 -0
- polyctl/adapters/openai-api.json +68 -0
- polyctl/adapters/opencode.json +162 -0
- polyctl/adapters/pi.json +24 -0
- polyctl/adapters/pr-agent.json +54 -0
- polyctl/adapters/profiles.json +9 -0
- polyctl/adapters/qoder.json +163 -0
- polyctl/adapters/qwen.json +27 -0
- polyctl/adapters/reasonix.json +25 -0
- polyctl/adapters/rewriters/rtk.json +24 -0
- polyctl/adapters/trae.json +31 -0
- polyctl/adapters/vendors.json +42 -0
- polyctl/adapters/warp.json +88 -0
- polyctl/allowlist.py +1756 -0
- polyctl/api_review.py +920 -0
- polyctl/cli.py +835 -0
- polyctl/conformance_harness.py +9 -0
- polyctl/demo.py +503 -0
- polyctl/drivers.py +146 -0
- polyctl/email.py +160 -0
- polyctl/features.py +98 -0
- polyctl/journal.py +144 -0
- polyctl/language.py +945 -0
- polyctl/lib.py +1296 -0
- polyctl/matrix.py +660 -0
- polyctl/mcp.py +1362 -0
- polyctl/ntfy.py +169 -0
- polyctl/outbound.py +39 -0
- polyctl/parts/__init__.py +5 -0
- polyctl/parts/cli_authority.py +2615 -0
- polyctl/parts/cli_channels.py +1325 -0
- polyctl/parts/cli_checks.py +2293 -0
- polyctl/parts/cli_completion.py +686 -0
- polyctl/parts/cli_evidence.py +141 -0
- polyctl/parts/cli_gain.py +375 -0
- polyctl/parts/cli_harness.py +392 -0
- polyctl/parts/cli_hooks.py +1816 -0
- polyctl/parts/cli_install.py +638 -0
- polyctl/parts/cli_lanes.py +106 -0
- polyctl/parts/cli_maintenance.py +2536 -0
- polyctl/parts/cli_next.py +248 -0
- polyctl/parts/cli_pr.py +53 -0
- polyctl/parts/cli_project.py +2857 -0
- polyctl/parts/cli_review.py +5158 -0
- polyctl/parts/cli_status.py +898 -0
- polyctl/parts/lib_adapters.py +2933 -0
- polyctl/parts/lib_alarms.py +2106 -0
- polyctl/parts/lib_events.py +224 -0
- polyctl/parts/lib_evidence.py +412 -0
- polyctl/parts/lib_gates.py +2099 -0
- polyctl/parts/lib_lanes.py +829 -0
- polyctl/parts/lib_mail.py +1541 -0
- polyctl/parts/lib_next.py +652 -0
- polyctl/parts/lib_pr.py +331 -0
- polyctl/parts/lib_slices.py +3337 -0
- polyctl/parts/lib_state.py +2415 -0
- polyctl/parts/lib_transcript.py +4118 -0
- polyctl/pressure.py +325 -0
- polyctl/prices.json +47 -0
- polyctl/procs.py +1194 -0
- polyctl/settings.py +505 -0
- polyctl/skill/SKILL.md +365 -0
- polyctl/skillkit.py +503 -0
- polyctl/storage.py +2055 -0
- polyctl/telegram.py +327 -0
- polyctl/tiers.py +123 -0
- polyctl/verdicts.py +41 -0
- polyctl/watchdog.py +3887 -0
- polyctl/web.py +337 -0
- polyctl-0.7.0.dist-info/METADATA +235 -0
- polyctl-0.7.0.dist-info/RECORD +99 -0
- polyctl-0.7.0.dist-info/WHEEL +4 -0
- polyctl-0.7.0.dist-info/entry_points.txt +5 -0
- polyctl-0.7.0.dist-info/licenses/LICENSE +21 -0
polyctl/storage.py
ADDED
|
@@ -0,0 +1,2055 @@
|
|
|
1
|
+
"""Crash-safe local persistence primitives for polyctl's append-only ledgers.
|
|
2
|
+
|
|
3
|
+
The ledgers are authority evidence, not logging. A caller may act on a row as
|
|
4
|
+
soon as this module returns, so returning means all bytes reached ``fsync`` and,
|
|
5
|
+
for a newly-created file, the containing directory entry did too. The API stays
|
|
6
|
+
standard-library-only and uses an OS lock to keep independent polyctl processes from
|
|
7
|
+
interleaving JSON records or selecting competing hash-chain predecessors.
|
|
8
|
+
"""
|
|
9
|
+
from contextlib import contextmanager
|
|
10
|
+
import errno
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import stat
|
|
16
|
+
import time
|
|
17
|
+
|
|
18
|
+
UTF8 = "utf-8"
|
|
19
|
+
CHAIN_PREVIOUS_FIELD = "previous"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class LedgerCorruption(RuntimeError):
|
|
23
|
+
"""A committed ledger record is malformed or violates its hash chain."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class LedgerLockTimeout(TimeoutError):
|
|
27
|
+
"""Another process held a ledger lock past the caller's deadline.
|
|
28
|
+
|
|
29
|
+
`path` is the lock and `timeout` the seconds waited for it, so a command can say in one
|
|
30
|
+
line which lock stood in its way (CLI-ROBUST).
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(self, message, path=None, timeout=None):
|
|
34
|
+
super().__init__(message)
|
|
35
|
+
self.path, self.timeout = path, timeout
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class LockThroughLink(OSError):
|
|
39
|
+
"""A lock file that is a link, or not a regular file: polyctl takes no lock through it (LOCK-LINKS)."""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _call(checkpoint, step):
|
|
43
|
+
if checkpoint is not None:
|
|
44
|
+
checkpoint(step)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _open_lock_nt(path, refused):
|
|
48
|
+
"""A Windows lock file's handle, opened as the link itself where the path is one, never through it (LOCK-LINKS).
|
|
49
|
+
|
|
50
|
+
FILE_FLAG_OPEN_REPARSE_POINT opens a link, not what it names, so OPEN_ALWAYS makes nothing through one; the
|
|
51
|
+
caller refuses the handle when it is a reparse point. A link Windows will not open that way is refused here.
|
|
52
|
+
"""
|
|
53
|
+
import ctypes
|
|
54
|
+
import msvcrt
|
|
55
|
+
import stat as _stat
|
|
56
|
+
from ctypes import wintypes
|
|
57
|
+
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
58
|
+
create, close = kernel32.CreateFileW, kernel32.CloseHandle
|
|
59
|
+
create.argtypes = (wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD,
|
|
60
|
+
wintypes.DWORD, wintypes.HANDLE)
|
|
61
|
+
create.restype = wintypes.HANDLE
|
|
62
|
+
close.argtypes = (wintypes.HANDLE,)
|
|
63
|
+
generic_read_write, share_read_write, open_always = 0x80000000 | 0x40000000, 0x1 | 0x2, 4
|
|
64
|
+
normal_open_reparse_point = 0x80 | 0x00200000
|
|
65
|
+
raw = create(path, generic_read_write, share_read_write, None, open_always, normal_open_reparse_point, None)
|
|
66
|
+
if raw is None or raw == wintypes.HANDLE(-1).value:
|
|
67
|
+
error = ctypes.WinError(ctypes.get_last_error())
|
|
68
|
+
try:
|
|
69
|
+
linked = _stat.S_ISLNK(os.lstat(path).st_mode)
|
|
70
|
+
except OSError:
|
|
71
|
+
linked = False
|
|
72
|
+
if linked:
|
|
73
|
+
raise LockThroughLink(refused) from error
|
|
74
|
+
raise error
|
|
75
|
+
try:
|
|
76
|
+
fd = msvcrt.open_osfhandle(raw, os.O_APPEND)
|
|
77
|
+
except BaseException:
|
|
78
|
+
close(raw)
|
|
79
|
+
raise
|
|
80
|
+
try:
|
|
81
|
+
return open(fd, "a+b", buffering=0)
|
|
82
|
+
except BaseException:
|
|
83
|
+
os.close(fd)
|
|
84
|
+
raise
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _open_lock(path):
|
|
88
|
+
"""A lock file's handle: the file reached through no final link, a regular file of one name; LockThroughLink
|
|
89
|
+
otherwise.
|
|
90
|
+
|
|
91
|
+
A lock was opened "a+b", which follows a final link and creates the file it names: a link put where a lock is
|
|
92
|
+
kept - `.polyctl/` in the working tree holds several - had polyctl create a file wherever the link pointed and write a
|
|
93
|
+
byte into it, outside the tree an agent may write. On POSIX a lock is opened with O_NOFOLLOW, and without
|
|
94
|
+
blocking, so a FIFO put in its place is refused at once rather than waited on; a POSIX platform without
|
|
95
|
+
O_NOFOLLOW takes no lock. Windows has no O_NOFOLLOW, and looking at the path before and after a creating open
|
|
96
|
+
left that open to follow a link put there in between: there the path is opened as the link itself
|
|
97
|
+
(`_open_lock_nt`), so nothing is made through one. A link, anything but a regular file, or a file that has
|
|
98
|
+
another name - a hard link put in the lock's place - is then refused before anything is written (LOCK-LINKS).
|
|
99
|
+
|
|
100
|
+
The way is chosen by what the platform offers - O_NOFOLLOW where it has one - and not by `os.name`, which a
|
|
101
|
+
test of Windows' behaviour sets on another platform: there the lock went the Windows way, to a kernel32 that
|
|
102
|
+
platform has not, and the hold the test ran failed where its lock was taken (LOCK-LINKS-2).
|
|
103
|
+
"""
|
|
104
|
+
import stat as _stat
|
|
105
|
+
refused = (f"{path} is a link, not a regular file, or a file with another name: polyctl takes no lock through it; "
|
|
106
|
+
"remove it, and polyctl makes the lock again")
|
|
107
|
+
nofollow = getattr(os, "O_NOFOLLOW", None)
|
|
108
|
+
if nofollow is None and os.name == "nt":
|
|
109
|
+
handle = _open_lock_nt(path, refused)
|
|
110
|
+
else:
|
|
111
|
+
if nofollow is None:
|
|
112
|
+
raise LockThroughLink(f"{path}: this platform opens no file through no link (it has no O_NOFOLLOW), "
|
|
113
|
+
"and polyctl takes no lock it cannot open that way")
|
|
114
|
+
flags = nofollow | getattr(os, "O_NONBLOCK", 0)
|
|
115
|
+
try:
|
|
116
|
+
handle = open(path, "a+b", buffering=0, opener=lambda name, mode: os.open(name, mode | flags, 0o666))
|
|
117
|
+
except OSError as exc:
|
|
118
|
+
if exc.errno in (errno.ELOOP, errno.EMLINK):
|
|
119
|
+
raise LockThroughLink(refused) from exc # what O_NOFOLLOW answers a final link with
|
|
120
|
+
raise
|
|
121
|
+
try:
|
|
122
|
+
opened = os.fstat(handle.fileno())
|
|
123
|
+
reparse = getattr(opened, "st_file_attributes", 0) & getattr(_stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
|
|
124
|
+
if not _stat.S_ISREG(opened.st_mode) or reparse or opened.st_nlink != 1:
|
|
125
|
+
raise LockThroughLink(refused)
|
|
126
|
+
except BaseException:
|
|
127
|
+
handle.close()
|
|
128
|
+
raise
|
|
129
|
+
return handle
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _lock_still_named(path, handle):
|
|
133
|
+
"""Whether `path` still names, through no link, the one file `handle` holds (LOCK-LINKS)."""
|
|
134
|
+
import stat as _stat
|
|
135
|
+
try:
|
|
136
|
+
at = os.lstat(path)
|
|
137
|
+
except FileNotFoundError:
|
|
138
|
+
return False
|
|
139
|
+
held = os.fstat(handle.fileno())
|
|
140
|
+
if _stat.S_ISLNK(at.st_mode) or held.st_nlink != 1:
|
|
141
|
+
return False
|
|
142
|
+
if not at.st_ino or not held.st_ino:
|
|
143
|
+
return True # a file system that numbers no file cannot tell, and has no links to swap in
|
|
144
|
+
return (at.st_dev, at.st_ino) == (held.st_dev, held.st_ino)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _let_go(handle):
|
|
148
|
+
try:
|
|
149
|
+
handle.seek(0)
|
|
150
|
+
if os.name == "nt":
|
|
151
|
+
import msvcrt
|
|
152
|
+
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
|
153
|
+
else:
|
|
154
|
+
import fcntl
|
|
155
|
+
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
|
156
|
+
except OSError:
|
|
157
|
+
pass
|
|
158
|
+
handle.close()
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@contextmanager
|
|
162
|
+
def _exclusive_lock(path, timeout=10.0, *, _folder=None):
|
|
163
|
+
"""Cross-platform advisory lock on one byte of a sidecar file.
|
|
164
|
+
|
|
165
|
+
The byte is written once, by whichever process finds the file empty. On Windows a lock is mandatory for
|
|
166
|
+
writes, and another process could take the file between this one's look and its write: the write then
|
|
167
|
+
failed with PermissionError, as four processes appending to a new ledger at once showed on the Windows
|
|
168
|
+
lane. The file is no longer empty then, which is all the write was for, and the handle buffers nothing,
|
|
169
|
+
so nothing is written again when it closes (WINDOWS-LOCK-WRITE).
|
|
170
|
+
|
|
171
|
+
A lock is held only while its path still names the file locked: a file whose name was taken away, or given
|
|
172
|
+
to another file, after it was opened - an agent writes `.polyctl/` - is let go and the path's file taken instead,
|
|
173
|
+
where two of polyctl's writers each held a file of the one name. A lock in the tree orders polyctl's own writers; an
|
|
174
|
+
agent that replaces one while it is held could already write the ledger it guards, which the ledger's chain
|
|
175
|
+
and checkpoints show (LOCK-LINKS).
|
|
176
|
+
"""
|
|
177
|
+
parent = os.path.dirname(path) or "."
|
|
178
|
+
if _folder is None:
|
|
179
|
+
os.makedirs(parent, exist_ok=True)
|
|
180
|
+
deadline = time.monotonic() + max(0.0, timeout)
|
|
181
|
+
while True:
|
|
182
|
+
if _folder is None:
|
|
183
|
+
handle = _open_lock(path)
|
|
184
|
+
else:
|
|
185
|
+
fd = _tree_open(_folder, os.path.basename(path), os.O_RDWR | os.O_CREAT | os.O_APPEND)
|
|
186
|
+
if os.fstat(fd).st_nlink != 1:
|
|
187
|
+
os.close(fd)
|
|
188
|
+
raise LockThroughLink(f"{path} has another name; remove the link before retrying")
|
|
189
|
+
handle = open(fd, "a+b", buffering=0)
|
|
190
|
+
try:
|
|
191
|
+
handle.seek(0, os.SEEK_END)
|
|
192
|
+
if handle.tell() == 0:
|
|
193
|
+
try:
|
|
194
|
+
handle.write(b"\0")
|
|
195
|
+
except PermissionError:
|
|
196
|
+
pass
|
|
197
|
+
while True:
|
|
198
|
+
try:
|
|
199
|
+
handle.seek(0)
|
|
200
|
+
if os.name == "nt":
|
|
201
|
+
import msvcrt
|
|
202
|
+
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
|
203
|
+
else:
|
|
204
|
+
import fcntl
|
|
205
|
+
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
206
|
+
break
|
|
207
|
+
except (OSError, BlockingIOError):
|
|
208
|
+
if time.monotonic() >= deadline:
|
|
209
|
+
raise LedgerLockTimeout(f"timed out locking {path}", path, timeout)
|
|
210
|
+
time.sleep(0.02)
|
|
211
|
+
if _folder is None:
|
|
212
|
+
named = _lock_still_named(path, handle)
|
|
213
|
+
else:
|
|
214
|
+
try:
|
|
215
|
+
at = _tree_stat(_folder, os.path.basename(path))
|
|
216
|
+
held = os.fstat(handle.fileno())
|
|
217
|
+
named = not stat.S_ISLNK(at.st_mode) and held.st_nlink == 1 and \
|
|
218
|
+
(at.st_dev, at.st_ino) == (held.st_dev, held.st_ino)
|
|
219
|
+
except FileNotFoundError:
|
|
220
|
+
named = False
|
|
221
|
+
except BaseException:
|
|
222
|
+
_let_go(handle)
|
|
223
|
+
raise
|
|
224
|
+
if named:
|
|
225
|
+
break
|
|
226
|
+
_let_go(handle)
|
|
227
|
+
if time.monotonic() >= deadline:
|
|
228
|
+
raise LedgerLockTimeout(f"timed out locking {path}: its file was replaced as it was taken", path,
|
|
229
|
+
timeout)
|
|
230
|
+
try:
|
|
231
|
+
yield
|
|
232
|
+
finally:
|
|
233
|
+
_let_go(handle)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _read_jsonl_unlocked(path, allow_partial_tail=True):
|
|
237
|
+
"""Parse a ledger while the caller owns its lock.
|
|
238
|
+
|
|
239
|
+
A row is committed by its newline. A final line without one - even valid
|
|
240
|
+
JSON - is what an interrupted or failed append leaves, and is never a row:
|
|
241
|
+
a grant whose append raised must not be read back as authority (#68).
|
|
242
|
+
"""
|
|
243
|
+
try:
|
|
244
|
+
data = open(path, "rb").read()
|
|
245
|
+
except FileNotFoundError:
|
|
246
|
+
return []
|
|
247
|
+
return _jsonl_rows(data, path, allow_partial_tail)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _jsonl_records(data):
|
|
251
|
+
"""([(line number, record bytes)] of each complete, non-empty line, the line number of an unfinished last one or
|
|
252
|
+
None) - the one way polyctl splits JSONL bytes into records, which a read, a count and an append all use (OPTIN-ATTESTED).
|
|
253
|
+
|
|
254
|
+
A line ends at `\n`, `\r` or `\r\n`; an empty one is no record, and one of blanks is a record a parse refuses.
|
|
255
|
+
Counted any other way, a row put after `\r` on polyctl's own line was one line to a count and two records to a read.
|
|
256
|
+
"""
|
|
257
|
+
records = []
|
|
258
|
+
for index, raw in enumerate(data.splitlines(keepends=True)):
|
|
259
|
+
body = raw.rstrip(b"\r\n")
|
|
260
|
+
if not body:
|
|
261
|
+
continue
|
|
262
|
+
if not raw.endswith((b"\n", b"\r")):
|
|
263
|
+
return records, index
|
|
264
|
+
records.append((index, body))
|
|
265
|
+
return records, None
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _jsonl_row(body, index, path):
|
|
269
|
+
try:
|
|
270
|
+
return json.loads(body.decode(UTF8))
|
|
271
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
272
|
+
raise LedgerCorruption(f"malformed JSONL record {index + 1} in {path}: {exc}") from exc
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _jsonl_rows(data, path, allow_partial_tail=True):
|
|
276
|
+
"""Parse bytes already read through a held file (LEADS-ADAPTERS)."""
|
|
277
|
+
records, unfinished = _jsonl_records(data)
|
|
278
|
+
rows = [_jsonl_row(body, index, path) for index, body in records]
|
|
279
|
+
if unfinished is not None and not allow_partial_tail:
|
|
280
|
+
raise LedgerCorruption(f"uncommitted final record {unfinished + 1} in {path}")
|
|
281
|
+
return rows
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def read_jsonl(path, allow_partial_tail=True, timeout=10.0):
|
|
285
|
+
"""Read committed JSONL records and fail on corruption before the tail.
|
|
286
|
+
|
|
287
|
+
Readers share the writer's sidecar lock so they cannot observe a complete
|
|
288
|
+
row in the interval after ``write`` but before ``fsync``. A killed process
|
|
289
|
+
may still leave the final line without its newline; readers may ignore that
|
|
290
|
+
one incomplete tail, while malformed complete rows always fail closed.
|
|
291
|
+
"""
|
|
292
|
+
if not os.path.exists(path):
|
|
293
|
+
return []
|
|
294
|
+
with _exclusive_lock(path + ".lock", timeout=timeout):
|
|
295
|
+
return _read_jsonl_unlocked(path, allow_partial_tail)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def chained_row_digest(record, chain):
|
|
299
|
+
"""Canonical, domain-separated SHA-256 digest of one chained JSON object."""
|
|
300
|
+
if not isinstance(record, dict):
|
|
301
|
+
raise TypeError("a chained ledger row must be a JSON object")
|
|
302
|
+
if not isinstance(chain, str) or not chain:
|
|
303
|
+
raise ValueError("a chained ledger requires a non-empty domain")
|
|
304
|
+
canonical = json.dumps(
|
|
305
|
+
record,
|
|
306
|
+
ensure_ascii=True,
|
|
307
|
+
allow_nan=False,
|
|
308
|
+
sort_keys=True,
|
|
309
|
+
separators=(",", ":"),
|
|
310
|
+
).encode("ascii")
|
|
311
|
+
domain = chain.encode(UTF8)
|
|
312
|
+
digest = hashlib.sha256()
|
|
313
|
+
digest.update(domain)
|
|
314
|
+
digest.update(b"\0")
|
|
315
|
+
digest.update(canonical)
|
|
316
|
+
return "sha256:" + digest.hexdigest()
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def checkpoint_path():
|
|
320
|
+
"""Where chained ledgers record their committed length: outside the repository (#62).
|
|
321
|
+
|
|
322
|
+
A hash chain detects a changed or inserted row but not a removed tail, because
|
|
323
|
+
every prefix of a valid chain is itself valid; deleting the newest grants, or
|
|
324
|
+
the whole file, left a ledger that passed every check. The recorded length is
|
|
325
|
+
kept apart from the ledger, so removing rows from the file does not remove the
|
|
326
|
+
record of how many there were.
|
|
327
|
+
"""
|
|
328
|
+
return (os.environ.get("POLYCTL_LEDGER_CHECKPOINTS")
|
|
329
|
+
or os.path.join(os.path.expanduser("~"), ".polyctl", "ledger-checkpoints.json"))
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _load_committed_lengths():
|
|
333
|
+
try:
|
|
334
|
+
with open(checkpoint_path(), encoding="utf-8") as handle:
|
|
335
|
+
data = json.load(handle)
|
|
336
|
+
except (OSError, ValueError):
|
|
337
|
+
return {}
|
|
338
|
+
return data if isinstance(data, dict) else {}
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
# What Windows answers, for a moment, when a rename would replace a file another handle
|
|
342
|
+
# holds open: ERROR_ACCESS_DENIED and ERROR_SHARING_VIOLATION.
|
|
343
|
+
_WINDOWS_IN_USE = (5, 32)
|
|
344
|
+
_REPLACE_ATTEMPTS = 8
|
|
345
|
+
_REPLACE_FIRST_WAIT = 0.01
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _replace(source, target):
|
|
349
|
+
"""``os.replace``, retried with backoff for about a second while Windows reports the target in use (#71).
|
|
350
|
+
|
|
351
|
+
A chained read opens the checkpoint store under its own ledger's lock, not the
|
|
352
|
+
store's, and Windows does not rename over a file another handle holds open: an
|
|
353
|
+
append to one ledger failed while another ledger was being read, and the row it had
|
|
354
|
+
written was taken back. A virus scanner holds a new file open the same way. Every
|
|
355
|
+
replace in this module comes here. Only those two Windows errors are retried, a
|
|
356
|
+
bounded number of times, and the last one still reaches the caller; POSIX renames
|
|
357
|
+
over an open file, and a PermissionError there is raised at once, as before.
|
|
358
|
+
"""
|
|
359
|
+
for attempt in range(_REPLACE_ATTEMPTS):
|
|
360
|
+
try:
|
|
361
|
+
return os.replace(source, target)
|
|
362
|
+
except PermissionError as exc:
|
|
363
|
+
if getattr(exc, "winerror", None) not in _WINDOWS_IN_USE or attempt + 1 == _REPLACE_ATTEMPTS:
|
|
364
|
+
raise
|
|
365
|
+
time.sleep(_REPLACE_FIRST_WAIT * 2 ** attempt)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _write_committed_lengths(data):
|
|
369
|
+
"""Replace the checkpoint store whole; the caller holds its lock.
|
|
370
|
+
|
|
371
|
+
Through replace_file_durably, under a name no file had and never through a link: `<store>.<pid>.tmp`, opened to
|
|
372
|
+
truncate, wrote the target of a link left at that name, and POLYCTL_LEDGER_CHECKPOINTS can name a store in a folder an
|
|
373
|
+
agent writes (STORAGE-TMP). It syncs the folder after the rename too: renamed into place with no sync, an append
|
|
374
|
+
returned on a record a power cut could still take back (the review of LEDGER-LENGTH, LEDGER-HISTORY).
|
|
375
|
+
"""
|
|
376
|
+
replace_file_durably(checkpoint_path(), json.dumps(data, indent=1, sort_keys=True).encode("utf-8"))
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _checkpoint_entry(path, *, identity=None):
|
|
380
|
+
"""What the checkpoint store records for one ledger: {} when nothing."""
|
|
381
|
+
entry = _load_committed_lengths().get(identity or os.path.realpath(path))
|
|
382
|
+
return entry if isinstance(entry, dict) else {}
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _update_checkpoint(path, change, *, identity=None):
|
|
386
|
+
"""Replace one ledger's record in the checkpoint store with `change(record)`, under the store's lock."""
|
|
387
|
+
store = checkpoint_path()
|
|
388
|
+
os.makedirs(os.path.dirname(store) or ".", exist_ok=True)
|
|
389
|
+
with _exclusive_lock(store + ".lock"):
|
|
390
|
+
data = _load_committed_lengths()
|
|
391
|
+
key = identity or os.path.realpath(path)
|
|
392
|
+
data[key] = change(dict(data[key]) if isinstance(data.get(key), dict) else {})
|
|
393
|
+
_write_committed_lengths(data)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _record_committed_length(path, count, digest, sealed=None, *, identity=None):
|
|
397
|
+
def change(previous):
|
|
398
|
+
mark = {"count": count, "digest": digest, "at": int(time.time())}
|
|
399
|
+
# A seal is recorded outside the repository too, so one written inside it
|
|
400
|
+
# cannot retire rows this machine never retired (#50).
|
|
401
|
+
if sealed is not None or previous.get("sealed"):
|
|
402
|
+
mark["sealed"] = sealed if sealed is not None else previous["sealed"]
|
|
403
|
+
# A seal under way is settled before any append; kept, one an append met anyway is refused, not lost.
|
|
404
|
+
if previous.get("intent"):
|
|
405
|
+
mark["intent"] = previous["intent"]
|
|
406
|
+
return mark
|
|
407
|
+
_update_checkpoint(path, change, identity=identity)
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _record_append_intent(path, digest, *, identity=None):
|
|
411
|
+
"""Record the digest of the row an append is about to write, cleared when its count is recorded (LEDGER-ATTESTED)."""
|
|
412
|
+
_update_checkpoint(path, lambda previous: dict(previous, append=digest), identity=identity)
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _clear_append_intent(path, *, identity=None):
|
|
416
|
+
"""Clear a write-ahead append intent once the uncommitted row has been taken back (LEDGER-ATTESTED)."""
|
|
417
|
+
_update_checkpoint(path, lambda previous: {k: v for k, v in previous.items() if k != "append"}, identity=identity)
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _check_committed_length(path, rows, chain, seal=None, *, identity=None):
|
|
421
|
+
"""Refuse a ledger shorter than its recorded length, or diverging from it (#62), across a seal (#50)."""
|
|
422
|
+
mark = _load_committed_lengths().get(identity or os.path.realpath(path))
|
|
423
|
+
if not isinstance(mark, dict) or not isinstance(mark.get("count"), int) or mark["count"] <= 0:
|
|
424
|
+
return
|
|
425
|
+
recorded = mark.get("sealed")
|
|
426
|
+
if recorded or seal:
|
|
427
|
+
same = bool(recorded and seal) and recorded.get("retired") == seal.get("retired") \
|
|
428
|
+
and recorded.get("digest") == seal.get("digest")
|
|
429
|
+
if not same:
|
|
430
|
+
raise LedgerCorruption(
|
|
431
|
+
f"{path} carries a seal this machine did not record, or lost the one it did "
|
|
432
|
+
f"(the record is kept in {checkpoint_path()})"
|
|
433
|
+
)
|
|
434
|
+
retired = seal["retired"] if seal else 0
|
|
435
|
+
count = mark["count"]
|
|
436
|
+
if retired + len(rows) < count:
|
|
437
|
+
raise LedgerCorruption(
|
|
438
|
+
f"{path} holds {retired + len(rows)} committed rows but recorded {count}: rows were removed from its end"
|
|
439
|
+
f" (the count is kept in {checkpoint_path()})"
|
|
440
|
+
)
|
|
441
|
+
if count <= retired:
|
|
442
|
+
if count == retired and seal.get("digest") == mark.get("digest"):
|
|
443
|
+
return
|
|
444
|
+
raise LedgerCorruption(f"{path} row {count} is sealed but does not match the digest recorded for it")
|
|
445
|
+
if chained_row_digest(rows[count - retired - 1], chain) != mark.get("digest"):
|
|
446
|
+
raise LedgerCorruption(
|
|
447
|
+
f"{path} row {count} does not match the digest recorded for it in {checkpoint_path()}"
|
|
448
|
+
)
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def seal_path(path):
|
|
452
|
+
return path + ".seal.json"
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _check_missing_chained_ledger(path, chain, seal=None, *, identity=None):
|
|
456
|
+
"""A file missing where a checkpoint or seal records rows is unreadable, never an empty history."""
|
|
457
|
+
_check_committed_length(path, [], chain, seal=seal, identity=identity)
|
|
458
|
+
entry = _checkpoint_entry(path, identity=identity)
|
|
459
|
+
if seal or entry.get("sealed") or "intent" in entry or \
|
|
460
|
+
(isinstance(entry.get("count"), int) and entry["count"] > 0):
|
|
461
|
+
raise LedgerCorruption(f"{path} is absent but its checkpoint or seal records rows: rows were removed; "
|
|
462
|
+
"restore the ledger before reading its history")
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _load_seal(path, *, _folder=None):
|
|
466
|
+
"""The seal that retired a ledger's prefix, or None (#50)."""
|
|
467
|
+
try:
|
|
468
|
+
if _folder is None:
|
|
469
|
+
with open(seal_path(path), encoding="utf-8") as handle:
|
|
470
|
+
seal = json.load(handle)
|
|
471
|
+
else:
|
|
472
|
+
data = _tree_bytes(_folder, os.path.basename(seal_path(path)))
|
|
473
|
+
if data is None:
|
|
474
|
+
return None
|
|
475
|
+
seal = json.loads(data.decode(UTF8))
|
|
476
|
+
except FileNotFoundError:
|
|
477
|
+
return None
|
|
478
|
+
except (OSError, ValueError) as exc:
|
|
479
|
+
raise LedgerCorruption(f"the seal of {path} cannot be read: {exc}") from exc
|
|
480
|
+
if not isinstance(seal, dict) or not isinstance(seal.get("retired"), int) or seal["retired"] < 1 \
|
|
481
|
+
or not isinstance(seal.get("digest"), str):
|
|
482
|
+
raise LedgerCorruption(f"the seal of {path} is malformed")
|
|
483
|
+
return seal
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _seal_intent(path, entry):
|
|
487
|
+
"""The seal of `path` under way that this machine recorded in `entry`, its checkpoint record, or None."""
|
|
488
|
+
intent = entry.get("intent")
|
|
489
|
+
if "intent" not in entry:
|
|
490
|
+
return None
|
|
491
|
+
before = intent.get("before") if isinstance(intent, dict) else None
|
|
492
|
+
if not isinstance(intent, dict) or not isinstance(intent.get("retired"), int) or intent["retired"] < 1 \
|
|
493
|
+
or not isinstance(intent.get("digest"), str) or not isinstance(intent.get("archives"), list) \
|
|
494
|
+
or not isinstance(before, dict) or not isinstance(before.get("count"), int) \
|
|
495
|
+
or before["count"] <= intent["retired"] \
|
|
496
|
+
or not isinstance(before.get("digest"), str):
|
|
497
|
+
raise LedgerCorruption(f"the seal of {path} under way, as {checkpoint_path()} records it, is malformed: "
|
|
498
|
+
"nothing is read from the ledger until a person compares the two")
|
|
499
|
+
return intent
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def _settle_seal(path, chain, previous_field, legacy_prefix=False, *, _folder=None, identity=None):
|
|
503
|
+
"""Finish or undo a seal a crash interrupted, from the intent this machine recorded; the caller holds the ledger lock (#50).
|
|
504
|
+
|
|
505
|
+
The intent was read from `<ledger>.seal.json.pending`, in the tree the implementer writes: a
|
|
506
|
+
ledger cut to its newest rows, beside an intent naming the cut rows as sealed, was finished
|
|
507
|
+
as a seal and the record outside the repository rewritten to match, so the cut, or a whole
|
|
508
|
+
chain of the writer's own with any genesis, read as authority (the review of BOUNDED-STORES).
|
|
509
|
+
A seal now records its intent in the checkpoint store, under that store's lock, before it
|
|
510
|
+
rewrites the ledger, and only that intent is settled: finished when the ledger starts after
|
|
511
|
+
the rows it retires, holds exactly the rows it kept and the archives hold exactly those it
|
|
512
|
+
retired; undone when the ledger is still the one it was taken from; refused otherwise. polyctl
|
|
513
|
+
writes no intent in the tree, so a `.pending` found there is none of its own: it is refused,
|
|
514
|
+
named and left where it is (LEDGER-HISTORY). The opt-in reader uses the same settlement,
|
|
515
|
+
with every tree operation relative to its held folder and its checkpoint identity fixed
|
|
516
|
+
before the folder can move (LEADS-ADAPTERS).
|
|
517
|
+
"""
|
|
518
|
+
stray = seal_path(path) + ".pending"
|
|
519
|
+
if _folder is None:
|
|
520
|
+
stray_exists = os.path.lexists(stray)
|
|
521
|
+
else:
|
|
522
|
+
try:
|
|
523
|
+
_tree_stat(_folder, os.path.basename(stray))
|
|
524
|
+
stray_exists = True
|
|
525
|
+
except FileNotFoundError:
|
|
526
|
+
stray_exists = False
|
|
527
|
+
if stray_exists:
|
|
528
|
+
raise LedgerCorruption(
|
|
529
|
+
f"{stray} names a seal of {os.path.basename(path)} that this machine did not begin: polyctl records a seal's "
|
|
530
|
+
f"intent in {checkpoint_path()}, never in the tree. It is left where it is and the ledger is not read "
|
|
531
|
+
"while it stands; read it, then move it out of .polyctl/ledger/"
|
|
532
|
+
)
|
|
533
|
+
entry = _checkpoint_entry(path, identity=identity)
|
|
534
|
+
intent = _seal_intent(path, entry)
|
|
535
|
+
if intent is None:
|
|
536
|
+
return
|
|
537
|
+
before = intent["before"]
|
|
538
|
+
if (entry.get("count"), entry.get("digest")) != (before["count"], before["digest"]):
|
|
539
|
+
raise LedgerCorruption(
|
|
540
|
+
f"the record of {path} in {checkpoint_path()} changed while a seal of it was under way: nothing is read "
|
|
541
|
+
"from the ledger until a person compares the two"
|
|
542
|
+
)
|
|
543
|
+
if intent.get("chain") != chain:
|
|
544
|
+
raise LedgerCorruption(f"the seal of {path} under way was begun for another chain than {chain!r}")
|
|
545
|
+
if _folder is None:
|
|
546
|
+
rows = _read_jsonl_unlocked(path, allow_partial_tail=False) if os.path.exists(path) else []
|
|
547
|
+
else:
|
|
548
|
+
rows = _jsonl_rows(_tree_bytes(_folder, os.path.basename(path)) or b"", path, allow_partial_tail=False)
|
|
549
|
+
if rows and isinstance(rows[0], dict) and rows[0].get(previous_field) == intent["digest"]:
|
|
550
|
+
# Rewritten: what stands must be the newest rows the intent kept, and the archives the rows it retired.
|
|
551
|
+
seal = {key: intent.get(key) for key in ("chain", "retired", "digest", "archives", "at")}
|
|
552
|
+
_validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
|
|
553
|
+
if intent["retired"] + len(rows) != before["count"] or chained_row_digest(rows[-1], chain) != before["digest"]:
|
|
554
|
+
raise LedgerCorruption(
|
|
555
|
+
f"{path} was rewritten by a seal and no longer holds the rows it kept, through row {before['count']}: "
|
|
556
|
+
"rows were changed or removed after it"
|
|
557
|
+
)
|
|
558
|
+
_, archived = _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix, _folder=_folder)
|
|
559
|
+
data = (json.dumps(seal, sort_keys=True) + "\n").encode(UTF8)
|
|
560
|
+
if _folder is None:
|
|
561
|
+
replace_file_durably(seal_path(path), data)
|
|
562
|
+
else:
|
|
563
|
+
_tree_replace_bytes(_folder, os.path.basename(seal_path(path)), data)
|
|
564
|
+
|
|
565
|
+
def finished(record):
|
|
566
|
+
record.pop("intent", None)
|
|
567
|
+
record.update(count=before["count"], digest=before["digest"], at=int(time.time()),
|
|
568
|
+
sealed={"retired": intent["retired"], "digest": intent["digest"], "archives": archived})
|
|
569
|
+
return record
|
|
570
|
+
_update_checkpoint(path, finished, identity=identity)
|
|
571
|
+
return
|
|
572
|
+
# Not rewritten: the ledger must still be the one the seal was taken from.
|
|
573
|
+
seal = _load_seal(path, _folder=_folder)
|
|
574
|
+
_validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
|
|
575
|
+
held = (seal["retired"] if seal else 0) + len(rows)
|
|
576
|
+
if held != before["count"] or not rows or chained_row_digest(rows[-1], chain) != before["digest"]:
|
|
577
|
+
raise LedgerCorruption(
|
|
578
|
+
f"{path} changed while a seal of it was under way: it holds {held} rows where the seal found "
|
|
579
|
+
f"{before['count']}"
|
|
580
|
+
)
|
|
581
|
+
_check_committed_length(path, rows, chain, seal=seal, identity=identity)
|
|
582
|
+
_sealed_rows_held(path, chain, seal, previous_field, legacy_prefix, _folder=_folder)
|
|
583
|
+
|
|
584
|
+
def undone(record):
|
|
585
|
+
record.pop("intent", None)
|
|
586
|
+
return record
|
|
587
|
+
_update_checkpoint(path, undone, identity=identity)
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def seal_chained_jsonl(path, chain, keep, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False,
|
|
591
|
+
_checkpoint=None):
|
|
592
|
+
"""Retire all but the newest `keep` rows of a chained ledger into an archive, never deleting one (#50).
|
|
593
|
+
|
|
594
|
+
The retired rows are written whole to `sealed/<ledger>.<first>-<last>.jsonl`
|
|
595
|
+
beside the ledger. The seal names how many rows it retired and the digest of
|
|
596
|
+
the last of them, which becomes the genesis the first kept row links to, and
|
|
597
|
+
the same seal is recorded outside the repository with the committed length.
|
|
598
|
+
A sealed ledger and the unsealed one give every reader the same newest rows.
|
|
599
|
+
Returns the seal, or None when there is nothing to retire.
|
|
600
|
+
|
|
601
|
+
Before the ledger is rewritten, the seal's intent - what it retires, the digest of
|
|
602
|
+
the last of those, its archives, and the length and newest digest the ledger had -
|
|
603
|
+
is recorded in the checkpoint store under that store's lock, and a seal cut short
|
|
604
|
+
is settled from that record alone (LEDGER-HISTORY). `_checkpoint` is the private
|
|
605
|
+
test seam the appends take.
|
|
606
|
+
"""
|
|
607
|
+
if keep < 1:
|
|
608
|
+
raise ValueError("a seal keeps at least one row")
|
|
609
|
+
with _exclusive_lock(path + ".lock"):
|
|
610
|
+
_settle_seal(path, chain, previous_field, legacy_prefix)
|
|
611
|
+
if not os.path.exists(path):
|
|
612
|
+
_check_missing_chained_ledger(path, chain, seal=_load_seal(path))
|
|
613
|
+
return None
|
|
614
|
+
rows = _read_jsonl_unlocked(path, allow_partial_tail=False)
|
|
615
|
+
seal = _load_seal(path)
|
|
616
|
+
_validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
|
|
617
|
+
_check_committed_length(path, rows, chain, seal=seal)
|
|
618
|
+
if len(rows) <= keep:
|
|
619
|
+
return None
|
|
620
|
+
retire, kept = rows[:-keep], rows[-keep:]
|
|
621
|
+
if any(not isinstance(row, dict) or previous_field not in row for row in kept):
|
|
622
|
+
return None # rows from before the chain cannot start a sealed ledger
|
|
623
|
+
# A seal adds its archive to ones that hold: stacked on a broken one, it would carry the break on.
|
|
624
|
+
_sealed_rows_held(path, chain, seal, previous_field, legacy_prefix)
|
|
625
|
+
base = seal["retired"] if seal else 0
|
|
626
|
+
first, last = base + 1, base + len(retire)
|
|
627
|
+
archive = os.path.join(os.path.dirname(path) or ".", "sealed", f"{os.path.basename(path)}.{first}-{last}.jsonl")
|
|
628
|
+
replace_file_durably(archive, b"".join(_encode_jsonl(row) for row in retire))
|
|
629
|
+
intent = {"chain": chain, "retired": last, "digest": chained_row_digest(retire[-1], chain),
|
|
630
|
+
"archives": list((seal or {}).get("archives") or []) + [os.path.basename(archive)],
|
|
631
|
+
"at": int(time.time()),
|
|
632
|
+
"before": {"count": base + len(rows), "digest": chained_row_digest(rows[-1], chain)}}
|
|
633
|
+
|
|
634
|
+
def begun(record):
|
|
635
|
+
record.update(count=intent["before"]["count"], digest=intent["before"]["digest"], at=intent["at"],
|
|
636
|
+
intent=intent)
|
|
637
|
+
return record
|
|
638
|
+
_update_checkpoint(path, begun)
|
|
639
|
+
_call(_checkpoint, "intent-recorded")
|
|
640
|
+
replace_file_durably(path, b"".join(_encode_jsonl(row) for row in kept))
|
|
641
|
+
_call(_checkpoint, "ledger-rewritten")
|
|
642
|
+
_settle_seal(path, chain, previous_field, legacy_prefix)
|
|
643
|
+
return _load_seal(path)
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix, recorded=None, *, _folder=None):
|
|
647
|
+
"""(the rows `seal` retired, oldest first, the digest of its archives' bytes), held to the seal (LEDGER-HISTORY).
|
|
648
|
+
|
|
649
|
+
The archives were read as they stood: a row edited in one was returned as it was, and a reader
|
|
650
|
+
took its id for one polyctl had recorded (the review of BOUNDED-STORES). They must now be one chain,
|
|
651
|
+
from the ledger's first row to the seal's digest, and hold exactly the rows it retired, or
|
|
652
|
+
LedgerCorruption names what does not hold. Archives whose bytes are those this machine held to
|
|
653
|
+
the seal when it sealed them - `recorded`, the digest the checkpoint store keeps - are not
|
|
654
|
+
walked link by link again: a reader of a long history pays for parsing it, not for its chain.
|
|
655
|
+
"""
|
|
656
|
+
if not seal:
|
|
657
|
+
return [], None
|
|
658
|
+
folder = os.path.join(os.path.dirname(path) or ".", "sealed")
|
|
659
|
+
names = seal.get("archives")
|
|
660
|
+
if not isinstance(names, list) or not names \
|
|
661
|
+
or not all(isinstance(name, str) and name and os.path.basename(name) == name for name in names):
|
|
662
|
+
raise LedgerCorruption(f"the seal of {path} names no archives polyctl can read")
|
|
663
|
+
parts, whole = [], hashlib.sha256()
|
|
664
|
+
for name in names:
|
|
665
|
+
archive = os.path.join(folder, name)
|
|
666
|
+
try:
|
|
667
|
+
if _folder is None:
|
|
668
|
+
with open(archive, "rb") as handle:
|
|
669
|
+
data = handle.read()
|
|
670
|
+
else:
|
|
671
|
+
with _tree_folder(_folder, ["sealed"], make=False) as held:
|
|
672
|
+
data = _tree_bytes(held, name) if held is not None else None
|
|
673
|
+
if data is None:
|
|
674
|
+
raise FileNotFoundError(archive)
|
|
675
|
+
except OSError as exc:
|
|
676
|
+
raise LedgerCorruption(f"{archive}, which the seal of {path} names, cannot be read: {exc}") from exc
|
|
677
|
+
whole.update(data)
|
|
678
|
+
parts.append((archive, data))
|
|
679
|
+
digest = "sha256:" + whole.hexdigest()
|
|
680
|
+
rows, expected = [], None
|
|
681
|
+
for archive, data in parts:
|
|
682
|
+
part = _jsonl_rows(data, archive, allow_partial_tail=False)
|
|
683
|
+
if digest != recorded:
|
|
684
|
+
_validate_chained_rows(archive, part, chain, previous_field, legacy_prefix and not rows,
|
|
685
|
+
seal={"digest": expected, "retired": len(rows)} if rows else None)
|
|
686
|
+
if part:
|
|
687
|
+
expected = chained_row_digest(part[-1], chain)
|
|
688
|
+
rows.extend(part)
|
|
689
|
+
if len(rows) != seal["retired"] or not rows or chained_row_digest(rows[-1], chain) != seal["digest"]:
|
|
690
|
+
raise LedgerCorruption(
|
|
691
|
+
f"the archives of {path} hold {len(rows)} rows ending at "
|
|
692
|
+
f"{chained_row_digest(rows[-1], chain) if rows else 'none'}, where its seal retired {seal['retired']} "
|
|
693
|
+
f"ending at {seal['digest']}"
|
|
694
|
+
)
|
|
695
|
+
return rows, digest
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def sealed_rows(path, chain, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False, seal=None):
|
|
699
|
+
"""Every row a seal retired from a ledger, oldest first, read from its archives and held to its seal (#50).
|
|
700
|
+
|
|
701
|
+
`seal` is the ledger's own when not given. The caller holds the ledger's lock, or reads a copy
|
|
702
|
+
no one appends to; `read_chained_history` reads the whole history under the lock (LEDGER-HISTORY).
|
|
703
|
+
"""
|
|
704
|
+
# Snapshot readers do not settle or write. They must refuse an unfinished seal rather than measure half of it.
|
|
705
|
+
stray = seal_path(path) + ".pending"
|
|
706
|
+
if os.path.lexists(stray):
|
|
707
|
+
raise LedgerCorruption(f"{stray} names a seal this machine did not begin: polyctl records its intent in "
|
|
708
|
+
f"{checkpoint_path()}, never in the tree; the ledger is not read while it stands")
|
|
709
|
+
if "intent" in _checkpoint_entry(path):
|
|
710
|
+
raise LedgerCorruption(f"the seal of {path} awaits settlement; read the ledger under its lock before "
|
|
711
|
+
"reading its archives")
|
|
712
|
+
seal = seal if seal is not None else _load_seal(path)
|
|
713
|
+
if not os.path.exists(path):
|
|
714
|
+
_check_missing_chained_ledger(path, chain, seal=seal)
|
|
715
|
+
return _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix)[0]
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def _validate_chained_rows(path, rows, chain, previous_field, legacy_prefix=False, seal=None):
|
|
719
|
+
expected = None
|
|
720
|
+
start = 0
|
|
721
|
+
offset = 0
|
|
722
|
+
if seal is not None:
|
|
723
|
+
# A sealed ledger's first row links to the last row the seal retired (#50).
|
|
724
|
+
expected, offset = seal["digest"], seal["retired"]
|
|
725
|
+
elif legacy_prefix:
|
|
726
|
+
# Rows written before a ledger was chained carry no link. They may only
|
|
727
|
+
# come first, and the first linked row names the digest of the last one.
|
|
728
|
+
while start < len(rows) and isinstance(rows[start], dict) \
|
|
729
|
+
and previous_field not in rows[start]:
|
|
730
|
+
start += 1
|
|
731
|
+
if start:
|
|
732
|
+
try:
|
|
733
|
+
expected = chained_row_digest(rows[start - 1], chain)
|
|
734
|
+
except (TypeError, ValueError) as exc:
|
|
735
|
+
raise LedgerCorruption(
|
|
736
|
+
f"cannot digest chained JSONL record {start} in {path}: {exc}"
|
|
737
|
+
) from exc
|
|
738
|
+
for index, row in enumerate(rows[start:], start + 1 + offset):
|
|
739
|
+
if not isinstance(row, dict):
|
|
740
|
+
raise LedgerCorruption(
|
|
741
|
+
f"chained JSONL record {index} in {path} is not an object"
|
|
742
|
+
)
|
|
743
|
+
if previous_field not in row:
|
|
744
|
+
raise LedgerCorruption(
|
|
745
|
+
f"chained JSONL record {index} in {path} has no {previous_field!r} field"
|
|
746
|
+
)
|
|
747
|
+
actual = row.get(previous_field)
|
|
748
|
+
if actual != expected:
|
|
749
|
+
raise LedgerCorruption(
|
|
750
|
+
f"broken hash chain at record {index} in {path}: "
|
|
751
|
+
f"expected {expected!r}, got {actual!r}"
|
|
752
|
+
)
|
|
753
|
+
if "ordinal" in row and row.get("ordinal") != index:
|
|
754
|
+
raise LedgerCorruption(
|
|
755
|
+
f"chained JSONL record {index} in {path} carries ordinal {row.get('ordinal')!r}"
|
|
756
|
+
)
|
|
757
|
+
try:
|
|
758
|
+
expected = chained_row_digest(row, chain)
|
|
759
|
+
except (TypeError, ValueError) as exc:
|
|
760
|
+
raise LedgerCorruption(
|
|
761
|
+
f"cannot digest chained JSONL record {index} in {path}: {exc}"
|
|
762
|
+
) from exc
|
|
763
|
+
return rows
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
# The gate ledgers: every chained ledger a gate decision reads, defined once for the attested-prefix rule and
|
|
767
|
+
# `polyctl doctor` (LEDGER-ATTESTED).
|
|
768
|
+
GATE_LEDGERS = (
|
|
769
|
+
("opt-ins", ".polyctl/ledger/opt-ins.jsonl", "polyctl-opt-in-row-v1", False, "no opt-in counts"),
|
|
770
|
+
("authority", ".polyctl/ledger/authority.jsonl", "polyctl-authority-row-v1", False, "no grant counts"),
|
|
771
|
+
("verifications", ".polyctl/ledger/verifications.jsonl", "polyctl-verification-row-v1", True, "no verification counts"),
|
|
772
|
+
("waivers", ".polyctl/ledger/waivers.jsonl", "polyctl-waiver-row-v1", True, "no waiver counts"),
|
|
773
|
+
("reviews", ".polyctl/ledger/reviews.jsonl", "polyctl-review-row-v1", False, "no review counts"),
|
|
774
|
+
("decisions", ".polyctl/ledger/decisions.jsonl", "polyctl-decision-row-v1", True, "no decision counts"),
|
|
775
|
+
("plans", ".polyctl/ledger/plans.jsonl", "polyctl-plan-row-v1", True, "no plan baseline counts"),
|
|
776
|
+
)
|
|
777
|
+
GATE_CHAINS = frozenset(chain for _, _, chain, _, _ in GATE_LEDGERS)
|
|
778
|
+
GATE_LEDGER_FILES = frozenset(os.path.basename(rel) for _, rel, _, _, _ in GATE_LEDGERS)
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def _is_gate_ledger(path, chain):
|
|
782
|
+
return chain in GATE_CHAINS or os.path.basename(path) in GATE_LEDGER_FILES
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def read_chained_jsonl(path, chain, allow_partial_tail=True, timeout=10.0, *,
|
|
786
|
+
previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False, lock=True, attested=False,
|
|
787
|
+
for_append=False):
|
|
788
|
+
"""Read and validate every committed row in one predecessor hash chain.
|
|
789
|
+
|
|
790
|
+
The chain proves no row was changed or inserted; the recorded length proves
|
|
791
|
+
none was cut off the end, including all of them (#62), and that for every
|
|
792
|
+
gate ledger (`GATE_LEDGERS`) every row returned is one this machine's
|
|
793
|
+
checkpoint store records polyctl appending (LEDGER-ATTESTED).
|
|
794
|
+
"""
|
|
795
|
+
if _is_gate_ledger(path, chain) and _ledger_store_in_tree(path):
|
|
796
|
+
if for_append:
|
|
797
|
+
raise LedgerCorruption(
|
|
798
|
+
f"the checkpoint store {checkpoint_path()} lies inside the project, where an agent writes: polyctl records "
|
|
799
|
+
"no row beside it - point POLYCTL_LEDGER_CHECKPOINTS outside the tree, or unset it")
|
|
800
|
+
return []
|
|
801
|
+
if not os.path.exists(path) and not os.path.lexists(seal_path(path) + ".pending") \
|
|
802
|
+
and "intent" not in _checkpoint_entry(path):
|
|
803
|
+
_check_missing_chained_ledger(path, chain, seal=_load_seal(path))
|
|
804
|
+
return []
|
|
805
|
+
if not lock:
|
|
806
|
+
return _read_chained_snapshot(
|
|
807
|
+
path, chain, allow_partial_tail, previous_field, legacy_prefix, attested=attested
|
|
808
|
+
)[0]
|
|
809
|
+
with _exclusive_lock(path + ".lock", timeout=timeout):
|
|
810
|
+
return _read_chained_unlocked(
|
|
811
|
+
path, chain, allow_partial_tail, previous_field, legacy_prefix, for_append=for_append
|
|
812
|
+
)[0]
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def _read_chained_unlocked(path, chain, allow_partial_tail, previous_field, legacy_prefix, *, every=False,
|
|
816
|
+
for_append=False):
|
|
817
|
+
"""(the live rows, the seal they follow), validated; the caller holds the lock."""
|
|
818
|
+
gate = _is_gate_ledger(path, chain)
|
|
819
|
+
if gate and _ledger_store_in_tree(path):
|
|
820
|
+
if for_append:
|
|
821
|
+
raise LedgerCorruption(
|
|
822
|
+
f"the checkpoint store {checkpoint_path()} lies inside the project, where an agent writes: polyctl records "
|
|
823
|
+
"no row beside it - point POLYCTL_LEDGER_CHECKPOINTS outside the tree, or unset it")
|
|
824
|
+
return (0 if every else []), None
|
|
825
|
+
_settle_seal(path, chain, previous_field, legacy_prefix)
|
|
826
|
+
seal = _load_seal(path)
|
|
827
|
+
if not os.path.exists(path):
|
|
828
|
+
_check_missing_chained_ledger(path, chain, seal=seal)
|
|
829
|
+
return (0 if every else []), seal
|
|
830
|
+
with open(path, "rb") as handle:
|
|
831
|
+
data = handle.read()
|
|
832
|
+
records, unfinished = _jsonl_records(data)
|
|
833
|
+
if every:
|
|
834
|
+
return len(records), seal
|
|
835
|
+
attested = _attested_length(path, seal) if gate else len(records)
|
|
836
|
+
if unfinished is not None and not allow_partial_tail:
|
|
837
|
+
raise LedgerCorruption(f"uncommitted final record {unfinished + 1} in {path}")
|
|
838
|
+
# For a gate ledger, only the rows polyctl recorded are read and checked: a tail past them - well formed or not - is
|
|
839
|
+
# nobody's record, and never makes polyctl's own unreadable (OPTIN-ATTESTED, LEDGER-ATTESTED).
|
|
840
|
+
rows = [_jsonl_row(body, index, path) for index, body in records[:attested]]
|
|
841
|
+
_validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
|
|
842
|
+
_check_committed_length(path, rows, chain, seal=seal)
|
|
843
|
+
if for_append and gate:
|
|
844
|
+
past = len(records) - len(rows)
|
|
845
|
+
if past > 0 and not _is_interrupted_append(records, len(rows), path, chain, unfinished=unfinished):
|
|
846
|
+
raise LedgerCorruption(f"{path}: {_unattested_repair(past, len(rows))}")
|
|
847
|
+
return rows, seal
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def _read_chained_snapshot(path, chain, allow_partial_tail, previous_field, legacy_prefix, *, attested=False):
|
|
851
|
+
"""(the live rows, the seal they follow, unattested info), validated without locking or settling a seal (EXPORT-OTLP)."""
|
|
852
|
+
stray = seal_path(path) + ".pending"
|
|
853
|
+
if os.path.lexists(stray):
|
|
854
|
+
raise LedgerCorruption(
|
|
855
|
+
f"{stray} names a seal of {os.path.basename(path)} that this machine did not begin: polyctl records a seal's "
|
|
856
|
+
f"intent in {checkpoint_path()}, never in the tree. It is left where it is and the ledger is not read "
|
|
857
|
+
"while it stands; read it, then move it out of .polyctl/ledger/"
|
|
858
|
+
)
|
|
859
|
+
entry = _checkpoint_entry(path)
|
|
860
|
+
if "intent" in entry:
|
|
861
|
+
raise LedgerCorruption(f"the seal of {path} awaits settlement; read the ledger under its lock before "
|
|
862
|
+
"reading a snapshot")
|
|
863
|
+
seal = _load_seal(path)
|
|
864
|
+
if not os.path.exists(path):
|
|
865
|
+
_check_missing_chained_ledger(path, chain, seal=seal)
|
|
866
|
+
return [], seal, None
|
|
867
|
+
rows = _read_jsonl_unlocked(path, allow_partial_tail)
|
|
868
|
+
_validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
|
|
869
|
+
_check_committed_length(path, rows, chain, seal=seal)
|
|
870
|
+
retired = seal["retired"] if seal else 0
|
|
871
|
+
total = retired + len(rows)
|
|
872
|
+
count = entry.get("count")
|
|
873
|
+
has_count = isinstance(count, int) and not isinstance(count, bool) and count > 0
|
|
874
|
+
unattested = None
|
|
875
|
+
if has_count:
|
|
876
|
+
if total > count:
|
|
877
|
+
extra = total - count
|
|
878
|
+
msg = (
|
|
879
|
+
f"{os.path.basename(path)} ({path}) holds {total} row(s), {extra} past its recorded count of {count} "
|
|
880
|
+
f"in {checkpoint_path()} (no checkpoint here for the trailing row(s): recorded elsewhere, reset, or "
|
|
881
|
+
"an append still under way); pass --attested to export only the attested part"
|
|
882
|
+
)
|
|
883
|
+
if not attested:
|
|
884
|
+
raise LedgerCorruption(msg)
|
|
885
|
+
rows = rows[:max(0, count - retired)]
|
|
886
|
+
unattested = {"ledger": os.path.basename(path), "reason": "rows_past_checkpoint", "message": msg}
|
|
887
|
+
elif total > 0 or seal is not None:
|
|
888
|
+
msg = (
|
|
889
|
+
f"{os.path.basename(path)} ({path}) holds {total} row(s) with no checkpoint here: moved, reset, "
|
|
890
|
+
f"recorded elsewhere, older than checkpoints (checkpoints are kept in {checkpoint_path()}); "
|
|
891
|
+
"pass --attested to export only the attested part"
|
|
892
|
+
)
|
|
893
|
+
if not attested:
|
|
894
|
+
raise LedgerCorruption(msg)
|
|
895
|
+
rows = []
|
|
896
|
+
seal = None
|
|
897
|
+
unattested = {"ledger": os.path.basename(path), "reason": "no_checkpoint", "message": msg}
|
|
898
|
+
else:
|
|
899
|
+
rows = []
|
|
900
|
+
return rows, seal, unattested
|
|
901
|
+
|
|
902
|
+
|
|
903
|
+
def _read_chained_history_snapshot(path, chain, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False,
|
|
904
|
+
attested=False):
|
|
905
|
+
"""(rows, unattested) for a lockless snapshot of a chained ledger's full history (EXPORT-OTLP)."""
|
|
906
|
+
if not os.path.exists(path) and not os.path.lexists(seal_path(path) + ".pending") \
|
|
907
|
+
and "intent" not in _checkpoint_entry(path):
|
|
908
|
+
_check_missing_chained_ledger(path, chain, seal=_load_seal(path))
|
|
909
|
+
return [], None
|
|
910
|
+
rows, seal, unattested = _read_chained_snapshot(
|
|
911
|
+
path, chain, True, previous_field, legacy_prefix, attested=attested
|
|
912
|
+
)
|
|
913
|
+
recorded = _checkpoint_entry(path).get("sealed")
|
|
914
|
+
retired, _ = _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix,
|
|
915
|
+
recorded=recorded.get("archives") if isinstance(recorded, dict) else None)
|
|
916
|
+
return retired + rows, unattested
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
def read_chained_history(path, chain, timeout=10.0, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False,
|
|
920
|
+
lock=True, attested=False):
|
|
921
|
+
"""Every row a chained ledger ever committed, oldest first: those its seal retired, then the live ones (LEDGER-HISTORY).
|
|
922
|
+
|
|
923
|
+
One chain, read under the ledger's lock: the archives end at the seal's digest with exactly
|
|
924
|
+
the rows it retired, and the live rows go on from it as `read_chained_jsonl` holds them. A
|
|
925
|
+
reader that asks what was ever granted, verified or waived reads this; one that asks for the
|
|
926
|
+
newest decision reads the live rows. Read from the live rows alone, a seal made a spent waiver
|
|
927
|
+
look unused and moved the first grant the landed-tree check starts from (the review of
|
|
928
|
+
BOUNDED-STORES). `lock=False` reads a snapshot without creating a `.lock` file or settling a
|
|
929
|
+
seal (EXPORT-OTLP).
|
|
930
|
+
"""
|
|
931
|
+
gate = _is_gate_ledger(path, chain)
|
|
932
|
+
if gate and _ledger_store_in_tree(path):
|
|
933
|
+
return []
|
|
934
|
+
if not os.path.exists(path) and not os.path.lexists(seal_path(path) + ".pending") \
|
|
935
|
+
and "intent" not in _checkpoint_entry(path):
|
|
936
|
+
_check_missing_chained_ledger(path, chain, seal=_load_seal(path))
|
|
937
|
+
return []
|
|
938
|
+
if not lock:
|
|
939
|
+
return _read_chained_history_snapshot(
|
|
940
|
+
path, chain, previous_field=previous_field, legacy_prefix=legacy_prefix, attested=attested
|
|
941
|
+
)[0]
|
|
942
|
+
with _exclusive_lock(path + ".lock", timeout=timeout):
|
|
943
|
+
rows, seal = _read_chained_unlocked(path, chain, True, previous_field, legacy_prefix)
|
|
944
|
+
recorded = _checkpoint_entry(path).get("sealed")
|
|
945
|
+
retired, _ = _sealed_rows_held(path, chain, seal if (not gate or isinstance(recorded, dict)) else None,
|
|
946
|
+
previous_field, legacy_prefix,
|
|
947
|
+
recorded=recorded.get("archives") if isinstance(recorded, dict) else None)
|
|
948
|
+
return retired + rows
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def unattested_ledger(path, chain, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False):
|
|
952
|
+
"""(rows this machine's checkpoint store does not record polyctl appending, rows it does) of a chained ledger
|
|
953
|
+
(LEDGER-ATTESTED)."""
|
|
954
|
+
if _ledger_store_in_tree(path):
|
|
955
|
+
if not os.path.exists(path):
|
|
956
|
+
return 0, 0
|
|
957
|
+
with _exclusive_lock(path + ".lock"):
|
|
958
|
+
with open(path, "rb") as handle:
|
|
959
|
+
return len(_jsonl_records(handle.read())[0]), 0
|
|
960
|
+
if not os.path.exists(path) and not os.path.lexists(seal_path(path) + ".pending") \
|
|
961
|
+
and "intent" not in _checkpoint_entry(path):
|
|
962
|
+
_check_missing_chained_ledger(path, chain, seal=_load_seal(path))
|
|
963
|
+
return 0, 0
|
|
964
|
+
with _exclusive_lock(path + ".lock"):
|
|
965
|
+
attested = len(_read_chained_unlocked(path, chain, True, previous_field, legacy_prefix)[0])
|
|
966
|
+
total = _read_chained_unlocked(path, chain, True, previous_field, legacy_prefix, every=True)[0]
|
|
967
|
+
return max(0, total - attested), attested
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def attest_restored_ledger(path, chain, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False, record=True):
|
|
971
|
+
"""Validate a restored chained ledger (and its seal and archives) and, when `record=True`, record this machine's
|
|
972
|
+
checkpoint for it so its rows count; returns `(count, digest, already_attested)` (LEDGER-ATTESTED)."""
|
|
973
|
+
if _ledger_store_in_tree(path):
|
|
974
|
+
raise LedgerCorruption(
|
|
975
|
+
f"the checkpoint store {checkpoint_path()} lies inside the project, where an agent writes: polyctl records "
|
|
976
|
+
"no row beside it - point POLYCTL_LEDGER_CHECKPOINTS outside the tree, or unset it")
|
|
977
|
+
with _exclusive_lock(path + ".lock"):
|
|
978
|
+
stray = seal_path(path) + ".pending"
|
|
979
|
+
if os.path.lexists(stray):
|
|
980
|
+
raise LedgerCorruption(f"{stray} names an unfinished seal in the restored backup")
|
|
981
|
+
seal = _load_seal(path)
|
|
982
|
+
_, archived = _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix)
|
|
983
|
+
rows = _read_jsonl_unlocked(path, allow_partial_tail=False) if os.path.exists(path) else []
|
|
984
|
+
_validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
|
|
985
|
+
retired = seal["retired"] if seal else 0
|
|
986
|
+
count = retired + len(rows)
|
|
987
|
+
digest = chained_row_digest(rows[-1], chain) if rows else (seal["digest"] if seal else None)
|
|
988
|
+
sealed = {"retired": seal["retired"], "digest": seal["digest"], "archives": archived} if seal else None
|
|
989
|
+
entry = _checkpoint_entry(path)
|
|
990
|
+
attested = (count == 0 or (entry.get("count") == count and entry.get("digest") == digest
|
|
991
|
+
and entry.get("sealed") == sealed and "intent" not in entry))
|
|
992
|
+
if record and count > 0:
|
|
993
|
+
mark = {"count": count, "digest": digest, "at": int(time.time())}
|
|
994
|
+
if sealed is not None:
|
|
995
|
+
mark["sealed"] = sealed
|
|
996
|
+
_update_checkpoint(path, lambda _: mark)
|
|
997
|
+
attested = True
|
|
998
|
+
return count, digest, attested
|
|
999
|
+
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
def _sync_directory(directory, fsync):
|
|
1003
|
+
"""Persist a newly-created directory entry where the platform supports it."""
|
|
1004
|
+
if os.name == "nt":
|
|
1005
|
+
return False
|
|
1006
|
+
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
|
|
1007
|
+
fd = os.open(directory or ".", flags)
|
|
1008
|
+
try:
|
|
1009
|
+
return _sync_folder_held(fd, fsync)
|
|
1010
|
+
finally:
|
|
1011
|
+
os.close(fd)
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
def _sync_folder_held(fd, fsync):
|
|
1015
|
+
"""Persist the entries of a directory held open as `fd`; False where its file system cannot."""
|
|
1016
|
+
try:
|
|
1017
|
+
fsync(fd)
|
|
1018
|
+
except OSError as exc:
|
|
1019
|
+
# Some filesystems expose directories but reject fsync. This is a
|
|
1020
|
+
# platform capability limit, not evidence that data was persisted.
|
|
1021
|
+
if exc.errno in (errno.EINVAL, errno.ENOTSUP, getattr(errno, "EOPNOTSUPP", -1)):
|
|
1022
|
+
return False
|
|
1023
|
+
raise
|
|
1024
|
+
return True
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
class ReplacedNotDurable(OSError):
|
|
1028
|
+
"""The replacement happened, but syncing its directory failed (TRIAGE-3-LOW)."""
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
def _write_all(fd, data):
|
|
1032
|
+
"""Write every byte of `data` to `fd`, or raise."""
|
|
1033
|
+
remaining = memoryview(data)
|
|
1034
|
+
while remaining:
|
|
1035
|
+
try:
|
|
1036
|
+
count = os.write(fd, remaining)
|
|
1037
|
+
except InterruptedError:
|
|
1038
|
+
continue
|
|
1039
|
+
if count <= 0:
|
|
1040
|
+
raise OSError(errno.EIO, "write made no forward progress")
|
|
1041
|
+
remaining = remaining[count:]
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def replace_file_durably(path, data, *, mode=None, before_replace=None, _checkpoint=None, _fsync=None):
|
|
1045
|
+
"""Replace a whole file so it is only ever seen with its old bytes or its new ones.
|
|
1046
|
+
|
|
1047
|
+
``open(path, "w")`` truncates first. A crash, kill or full disk before the new
|
|
1048
|
+
bytes are down leaves a zero-byte file, and a zero-byte ``.polyctl/config.json`` no
|
|
1049
|
+
longer holds the ``capability_matrix`` that keeps strict authority on (#56).
|
|
1050
|
+
This writes a temporary file beside the target, fsyncs it, renames it over the
|
|
1051
|
+
target and fsyncs the directory. ``_checkpoint`` and ``_fsync`` are the private
|
|
1052
|
+
test seams the ledger appends take.
|
|
1053
|
+
|
|
1054
|
+
``before_replace`` is called once the new bytes are durable beside the target and
|
|
1055
|
+
right before the rename: the last moment a caller that must not replace what
|
|
1056
|
+
someone changed since its reading can look, and what it raises leaves the target
|
|
1057
|
+
as it was and takes the temporary file away (SLICE-LIFECYCLE-2).
|
|
1058
|
+
|
|
1059
|
+
``mode`` is for a file that holds a credential. Its temporary file is created
|
|
1060
|
+
with that mode, under a name no file had (``O_EXCL``, and ``O_NOFOLLOW`` where
|
|
1061
|
+
the platform has it), so the bytes are never readable by anyone the mode leaves
|
|
1062
|
+
out - not while they are written, and not when a write is cut short - and a file
|
|
1063
|
+
or a link already at that name is never written through. Written in place and
|
|
1064
|
+
chmod-ed after, ``~/.polyctl/email.json`` was readable by every user of the machine
|
|
1065
|
+
meanwhile, and stayed so when the process stopped between the two
|
|
1066
|
+
(HUNT-LEADS-SEC). Without ``mode`` the file keeps the mode it had, or 0644.
|
|
1067
|
+
|
|
1068
|
+
Every temporary file is made that way, with or without ``mode``. Without one it
|
|
1069
|
+
was ``.<name>.<pid>.tmp``, opened to truncate and through any link: an agent that
|
|
1070
|
+
left a link at that name for a likely pid in a folder polyctl writes - ``.polyctl/`` among
|
|
1071
|
+
them - had polyctl truncate and write the link's target, a file of the person's
|
|
1072
|
+
(STORAGE-TMP).
|
|
1073
|
+
"""
|
|
1074
|
+
fsync = _fsync or os.fsync
|
|
1075
|
+
parent = os.path.dirname(path) or "."
|
|
1076
|
+
os.makedirs(parent, exist_ok=True)
|
|
1077
|
+
if mode is None:
|
|
1078
|
+
try:
|
|
1079
|
+
mode = os.stat(path).st_mode & 0o7777
|
|
1080
|
+
except OSError:
|
|
1081
|
+
mode = 0o644
|
|
1082
|
+
temporary = os.path.join(parent, f".{os.path.basename(path)}.{os.getpid()}.{os.urandom(4).hex()}.tmp")
|
|
1083
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_BINARY", 0)
|
|
1084
|
+
opened = False
|
|
1085
|
+
try:
|
|
1086
|
+
fd = os.open(temporary, flags, mode)
|
|
1087
|
+
opened = True
|
|
1088
|
+
try:
|
|
1089
|
+
_call(_checkpoint, "temporary-opened")
|
|
1090
|
+
_write_all(fd, data)
|
|
1091
|
+
_call(_checkpoint, "temporary-written")
|
|
1092
|
+
fsync(fd)
|
|
1093
|
+
_call(_checkpoint, "temporary-fsynced")
|
|
1094
|
+
finally:
|
|
1095
|
+
os.close(fd)
|
|
1096
|
+
if before_replace is not None:
|
|
1097
|
+
before_replace()
|
|
1098
|
+
_replace(temporary, path)
|
|
1099
|
+
except BaseException:
|
|
1100
|
+
# Only a file this call made goes: what stood at an exclusive name before it is not this call's.
|
|
1101
|
+
if opened:
|
|
1102
|
+
try:
|
|
1103
|
+
os.remove(temporary)
|
|
1104
|
+
except OSError:
|
|
1105
|
+
pass
|
|
1106
|
+
raise
|
|
1107
|
+
_call(_checkpoint, "replaced")
|
|
1108
|
+
try:
|
|
1109
|
+
synced = _sync_directory(parent, fsync)
|
|
1110
|
+
except OSError as exc:
|
|
1111
|
+
raise ReplacedNotDurable(f"{path} changed, but durability could not be confirmed ({exc}); "
|
|
1112
|
+
"read the file and retry after fixing the storage error") from exc
|
|
1113
|
+
if synced:
|
|
1114
|
+
_call(_checkpoint, "directory-fsynced")
|
|
1115
|
+
|
|
1116
|
+
|
|
1117
|
+
# ---- a file in a project's tree, written and read through no link (LEADS-ADAPTERS) -------------------------------
|
|
1118
|
+
#
|
|
1119
|
+
# `polyctl init` writes into a tree an agent writes too, and it wrote by path: each folder on the way was whatever stood
|
|
1120
|
+
# there as the file was opened. A `.polyctl` or an `agent-mail` that was a link - or became one between init's look and its
|
|
1121
|
+
# write - took the configuration, the board, the playbook and the manifest out of the project, and a `.polyctl/PLAYBOOK.md`
|
|
1122
|
+
# or an MCP file that was a link had polyctl's text added to the file of the person's it led to. Where the platform opens a
|
|
1123
|
+
# file relative to a folder it holds open, as POSIX does, each folder is opened so from the project's root down, never
|
|
1124
|
+
# through a link, and the file is made, replaced or read in the last of them: a link put on the way meanwhile moves
|
|
1125
|
+
# nothing. Windows holds every folder handle without delete sharing, so no folder can be renamed or replaced while
|
|
1126
|
+
# its path is used; every handle is opened as the reparse point itself and reparse points are refused.
|
|
1127
|
+
|
|
1128
|
+
class InTheWay(OSError):
|
|
1129
|
+
"""What stands where a file is written or read in a project's tree and is not gone through: a link or a file where
|
|
1130
|
+
a folder goes, a link at the file's own name, or no regular file. `where` is its path in the tree (LEADS-ADAPTERS).
|
|
1131
|
+
"""
|
|
1132
|
+
|
|
1133
|
+
def __init__(self, where, what):
|
|
1134
|
+
super().__init__(f"{where} {what}")
|
|
1135
|
+
self.where = where
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
_A_FOLDER = "is a link or a file where a folder of the project's own goes"
|
|
1139
|
+
_A_LINK = "is a link, and polyctl reads and writes through none"
|
|
1140
|
+
_NO_FILE = "is no regular file"
|
|
1141
|
+
_WINDOWS_LINK_TAG = 0x20000000 # the name-surrogate bit of a reparse tag: a symbolic link's, a junction's
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
def folders_held_open():
|
|
1145
|
+
"""Whether this platform opens a file relative to a folder it holds open, as POSIX does and Python on Windows does
|
|
1146
|
+
not (LEADS-ADAPTERS)."""
|
|
1147
|
+
return bool(getattr(os, "O_NOFOLLOW", 0) and getattr(os, "O_DIRECTORY", 0)) and \
|
|
1148
|
+
{os.open, os.mkdir, os.rename, os.stat, os.unlink} <= os.supports_dir_fd
|
|
1149
|
+
|
|
1150
|
+
|
|
1151
|
+
def not_a_folder_of_its_own(path):
|
|
1152
|
+
"""Whether what stands at `path` is no folder of the project's own: a link, a junction - Windows' link to a folder,
|
|
1153
|
+
which `os.path.islink` does not name before Python 3.12, known by the name-surrogate bit of its reparse tag, which a
|
|
1154
|
+
cloud file's placeholder does not carry - a file, or what cannot be looked at (LEADS-ADAPTERS)."""
|
|
1155
|
+
try:
|
|
1156
|
+
info = os.lstat(path)
|
|
1157
|
+
except FileNotFoundError:
|
|
1158
|
+
return False
|
|
1159
|
+
except OSError:
|
|
1160
|
+
return True
|
|
1161
|
+
return stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode) \
|
|
1162
|
+
or bool(getattr(info, "st_reparse_tag", 0) & _WINDOWS_LINK_TAG)
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
def _tree_parts(rel):
|
|
1166
|
+
"""The names of `rel`, a path within a project, folder by folder."""
|
|
1167
|
+
text = str(rel).replace(os.sep, "/")
|
|
1168
|
+
parts = [part for part in text.split("/") if part not in ("", ".")]
|
|
1169
|
+
if not parts or ".." in parts or (os.name == "nt" and any(":" in part for part in parts)):
|
|
1170
|
+
raise InTheWay(text, "names no file within the project")
|
|
1171
|
+
return parts
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
def _folder_held(root, parts, make):
|
|
1175
|
+
"""The descriptor of the folder `parts` names under `root`, each opened relative to the one before it and never
|
|
1176
|
+
through a link, and those missing made where `make`; None where one is missing. InTheWay where one is a link or a
|
|
1177
|
+
file."""
|
|
1178
|
+
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
1179
|
+
fd = os.dup(root) if isinstance(root, int) else os.open(root, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
1180
|
+
try:
|
|
1181
|
+
for at, part in enumerate(parts):
|
|
1182
|
+
if make:
|
|
1183
|
+
try:
|
|
1184
|
+
os.mkdir(part, dir_fd=fd)
|
|
1185
|
+
except FileExistsError:
|
|
1186
|
+
pass
|
|
1187
|
+
try:
|
|
1188
|
+
inner = os.open(part, flags, dir_fd=fd)
|
|
1189
|
+
except FileNotFoundError:
|
|
1190
|
+
if make:
|
|
1191
|
+
raise
|
|
1192
|
+
os.close(fd)
|
|
1193
|
+
return None
|
|
1194
|
+
except OSError as exc:
|
|
1195
|
+
if exc.errno in (errno.ELOOP, errno.ENOTDIR, errno.EMLINK):
|
|
1196
|
+
raise InTheWay("/".join(parts[:at + 1]), _A_FOLDER) from None
|
|
1197
|
+
raise
|
|
1198
|
+
held, fd = fd, inner
|
|
1199
|
+
os.close(held)
|
|
1200
|
+
except BaseException:
|
|
1201
|
+
os.close(fd)
|
|
1202
|
+
raise
|
|
1203
|
+
return fd
|
|
1204
|
+
|
|
1205
|
+
|
|
1206
|
+
def _windows_tree_available():
|
|
1207
|
+
"""Select the native handle path by its APIs, even when a test changes os.name (LEADS-ADAPTERS)."""
|
|
1208
|
+
import ctypes
|
|
1209
|
+
return callable(getattr(ctypes, "WinDLL", None))
|
|
1210
|
+
|
|
1211
|
+
|
|
1212
|
+
def _require_windows_tree():
|
|
1213
|
+
if not _windows_tree_available():
|
|
1214
|
+
raise InTheWay("this platform", "cannot hold folders through no link; polyctl refuses this file operation")
|
|
1215
|
+
|
|
1216
|
+
|
|
1217
|
+
def open_new_text(path):
|
|
1218
|
+
"""Create a UTF-8 file through no final link, or report that its name is taken (WINDOWS-TESTS-7).
|
|
1219
|
+
|
|
1220
|
+
Windows' CRT exclusive open follows a dangling link before creating its target. CREATE_NEW with
|
|
1221
|
+
OPEN_REPARSE_POINT instead refuses the occupied name, including a link planted during the open.
|
|
1222
|
+
Choose by the available APIs: tests can name a POSIX host Windows without giving it WinDLL.
|
|
1223
|
+
"""
|
|
1224
|
+
nofollow = getattr(os, "O_NOFOLLOW", None)
|
|
1225
|
+
if nofollow is not None:
|
|
1226
|
+
opener = lambda name, flags: os.open(name, flags | nofollow, 0o666)
|
|
1227
|
+
elif _windows_tree_available():
|
|
1228
|
+
opener = _open_tree_nt
|
|
1229
|
+
else:
|
|
1230
|
+
raise OSError(f"{path}: this platform cannot create a file through no link; "
|
|
1231
|
+
"polyctl refuses the write; use a supported platform")
|
|
1232
|
+
return open(path, "x", encoding=UTF8, opener=opener)
|
|
1233
|
+
|
|
1234
|
+
|
|
1235
|
+
def _open_tree_nt(path, flags=None, mode=0o666):
|
|
1236
|
+
"""Open the directory or leaf itself, with no delete sharing; reject every reparse point (LEADS-ADAPTERS)."""
|
|
1237
|
+
import ctypes
|
|
1238
|
+
import msvcrt
|
|
1239
|
+
from ctypes import wintypes
|
|
1240
|
+
|
|
1241
|
+
class Information(ctypes.Structure):
|
|
1242
|
+
_fields_ = [("attributes", wintypes.DWORD), ("created", wintypes.FILETIME),
|
|
1243
|
+
("accessed", wintypes.FILETIME), ("written", wintypes.FILETIME),
|
|
1244
|
+
("volume", wintypes.DWORD), ("size_high", wintypes.DWORD), ("size_low", wintypes.DWORD),
|
|
1245
|
+
("links", wintypes.DWORD), ("index_high", wintypes.DWORD), ("index_low", wintypes.DWORD)]
|
|
1246
|
+
|
|
1247
|
+
kernel = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
1248
|
+
create, close, information = kernel.CreateFileW, kernel.CloseHandle, kernel.GetFileInformationByHandle
|
|
1249
|
+
create.argtypes = (wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, wintypes.LPVOID,
|
|
1250
|
+
wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE)
|
|
1251
|
+
create.restype = wintypes.HANDLE
|
|
1252
|
+
close.argtypes = (wintypes.HANDLE,)
|
|
1253
|
+
close.restype = wintypes.BOOL
|
|
1254
|
+
information.argtypes = (wintypes.HANDLE, ctypes.POINTER(Information))
|
|
1255
|
+
information.restype = wintypes.BOOL
|
|
1256
|
+
directory = flags is None
|
|
1257
|
+
# A folder is opened to list it, not for its attributes alone: Windows records the share mode of a handle that
|
|
1258
|
+
# asks for data access only, so one that asked for none let another process rename the folder it was holding.
|
|
1259
|
+
access = (0x0001 | 0x0080) if directory else (0xC0000000 if flags & os.O_RDWR else
|
|
1260
|
+
0x40000000 if flags & os.O_WRONLY else 0x80000000)
|
|
1261
|
+
disposition = 3 if directory or not flags & os.O_CREAT else (1 if flags & os.O_EXCL else 4)
|
|
1262
|
+
raw = create(os.fspath(path), access, 0x1 | 0x2, None, disposition, 0x02000000 | 0x00200000, None)
|
|
1263
|
+
if raw is None or raw == wintypes.HANDLE(-1).value:
|
|
1264
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
1265
|
+
try:
|
|
1266
|
+
info = Information()
|
|
1267
|
+
if not information(raw, ctypes.byref(info)):
|
|
1268
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
1269
|
+
if info.attributes & 0x400:
|
|
1270
|
+
raise InTheWay(path, _A_FOLDER if directory else _A_LINK)
|
|
1271
|
+
if bool(info.attributes & 0x10) != directory:
|
|
1272
|
+
raise InTheWay(path, _A_FOLDER if directory else _NO_FILE)
|
|
1273
|
+
if directory:
|
|
1274
|
+
return raw, close
|
|
1275
|
+
fd = msvcrt.open_osfhandle(raw, flags & (os.O_APPEND | os.O_RDONLY | os.O_WRONLY | os.O_RDWR)
|
|
1276
|
+
| getattr(os, "O_BINARY", 0))
|
|
1277
|
+
except BaseException:
|
|
1278
|
+
close(raw)
|
|
1279
|
+
raise
|
|
1280
|
+
return fd
|
|
1281
|
+
|
|
1282
|
+
|
|
1283
|
+
@contextmanager
|
|
1284
|
+
def _folders_nt(root, parts, make):
|
|
1285
|
+
"""Pin the whole absolute path until the leaf operation ends, including the project's ancestors."""
|
|
1286
|
+
_require_windows_tree()
|
|
1287
|
+
absolute = os.path.abspath(root)
|
|
1288
|
+
drive, tail = os.path.splitdrive(absolute)
|
|
1289
|
+
folder = drive + os.sep
|
|
1290
|
+
ancestors = [part for part in tail.split(os.sep) if part]
|
|
1291
|
+
names = ancestors + list(parts)
|
|
1292
|
+
handles = []
|
|
1293
|
+
try:
|
|
1294
|
+
handles.append(_open_tree_nt(folder))
|
|
1295
|
+
for at, part in enumerate(names):
|
|
1296
|
+
folder = os.path.join(folder, part)
|
|
1297
|
+
# Ancestors and root must already exist; traversal permission need not allow mkdir there
|
|
1298
|
+
# (the round-4 review of LEADS-ADAPTERS).
|
|
1299
|
+
if make and at >= len(ancestors):
|
|
1300
|
+
try:
|
|
1301
|
+
os.mkdir(folder)
|
|
1302
|
+
except FileExistsError:
|
|
1303
|
+
pass
|
|
1304
|
+
try:
|
|
1305
|
+
handles.append(_open_tree_nt(folder))
|
|
1306
|
+
except FileNotFoundError:
|
|
1307
|
+
if make:
|
|
1308
|
+
raise
|
|
1309
|
+
yield None
|
|
1310
|
+
return
|
|
1311
|
+
except OSError as exc:
|
|
1312
|
+
shown = "/".join(parts[:at - len(ancestors) + 1]) if at >= len(ancestors) else folder
|
|
1313
|
+
raise InTheWay(shown, f"{_A_FOLDER}, or cannot be held ({exc})") from exc
|
|
1314
|
+
yield folder
|
|
1315
|
+
finally:
|
|
1316
|
+
for handle, close in reversed(handles):
|
|
1317
|
+
close(handle)
|
|
1318
|
+
|
|
1319
|
+
|
|
1320
|
+
@contextmanager
|
|
1321
|
+
def _tree_folder(root, parts, make):
|
|
1322
|
+
"""A dir_fd on POSIX or a path pinned by native handles on Windows (LEADS-ADAPTERS)."""
|
|
1323
|
+
if folders_held_open():
|
|
1324
|
+
fd = _folder_held(root, parts, make)
|
|
1325
|
+
try:
|
|
1326
|
+
yield fd
|
|
1327
|
+
finally:
|
|
1328
|
+
if fd is not None:
|
|
1329
|
+
os.close(fd)
|
|
1330
|
+
else:
|
|
1331
|
+
with _folders_nt(root, parts, make) as folder:
|
|
1332
|
+
yield folder
|
|
1333
|
+
|
|
1334
|
+
|
|
1335
|
+
@contextmanager
|
|
1336
|
+
def _tree_child(folder, name):
|
|
1337
|
+
"""Hold a child for reading or removal while its parent stays held, through no link (LEADS-INSTALL)."""
|
|
1338
|
+
if isinstance(folder, int):
|
|
1339
|
+
fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=folder)
|
|
1340
|
+
try:
|
|
1341
|
+
yield fd
|
|
1342
|
+
finally:
|
|
1343
|
+
os.close(fd)
|
|
1344
|
+
else:
|
|
1345
|
+
_require_windows_tree()
|
|
1346
|
+
child = os.path.join(folder, name)
|
|
1347
|
+
handle, close = _open_tree_nt(child)
|
|
1348
|
+
try:
|
|
1349
|
+
yield child
|
|
1350
|
+
finally:
|
|
1351
|
+
close(handle)
|
|
1352
|
+
|
|
1353
|
+
|
|
1354
|
+
def _tree_rmdir(folder, name):
|
|
1355
|
+
if isinstance(folder, int):
|
|
1356
|
+
os.rmdir(name, dir_fd=folder)
|
|
1357
|
+
else:
|
|
1358
|
+
os.rmdir(os.path.join(folder, name))
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
def _tree_remove_leaf(folder, name):
|
|
1362
|
+
"""Unlink a leaf, including a Windows junction itself, never its target (LEADS-INSTALL)."""
|
|
1363
|
+
info = _tree_stat(folder, name)
|
|
1364
|
+
if not isinstance(folder, int) and getattr(info, "st_file_attributes", 0) & 0x10:
|
|
1365
|
+
_tree_rmdir(folder, name)
|
|
1366
|
+
else:
|
|
1367
|
+
_tree_unlink(folder, name)
|
|
1368
|
+
|
|
1369
|
+
|
|
1370
|
+
def _tree_open(folder, name, flags, mode=0o666):
|
|
1371
|
+
if isinstance(folder, int):
|
|
1372
|
+
try:
|
|
1373
|
+
fd = os.open(name, flags | os.O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0), mode, dir_fd=folder)
|
|
1374
|
+
except OSError as exc:
|
|
1375
|
+
if exc.errno in (errno.ELOOP, errno.EMLINK):
|
|
1376
|
+
raise InTheWay(name, _A_LINK) from exc
|
|
1377
|
+
if exc.errno in (errno.ENXIO, errno.ENODEV, getattr(errno, "EOPNOTSUPP", -1)):
|
|
1378
|
+
raise InTheWay(name, _NO_FILE) from exc
|
|
1379
|
+
raise
|
|
1380
|
+
else:
|
|
1381
|
+
try:
|
|
1382
|
+
fd = _open_tree_nt(os.path.join(folder, name), flags, mode)
|
|
1383
|
+
except InTheWay as exc:
|
|
1384
|
+
raise InTheWay(name, str(exc)[len(str(exc.where)) + 1:]) from exc
|
|
1385
|
+
try:
|
|
1386
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
1387
|
+
raise InTheWay(name, _NO_FILE)
|
|
1388
|
+
except BaseException:
|
|
1389
|
+
os.close(fd)
|
|
1390
|
+
raise
|
|
1391
|
+
return fd
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
def _tree_stat(folder, name):
|
|
1395
|
+
return os.stat(name, dir_fd=folder, follow_symlinks=False) if isinstance(folder, int) else \
|
|
1396
|
+
os.lstat(os.path.join(folder, name))
|
|
1397
|
+
|
|
1398
|
+
|
|
1399
|
+
def _tree_unlink(folder, name):
|
|
1400
|
+
if isinstance(folder, int):
|
|
1401
|
+
os.unlink(name, dir_fd=folder)
|
|
1402
|
+
else:
|
|
1403
|
+
os.unlink(os.path.join(folder, name))
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def _tree_sync(folder, fsync):
|
|
1407
|
+
return _sync_folder_held(folder, fsync) if isinstance(folder, int) else False
|
|
1408
|
+
|
|
1409
|
+
|
|
1410
|
+
def create_in_tree(root, rel, data):
|
|
1411
|
+
"""Make the file `rel` in the project at `root`, holding `data`, where nothing stands at that name: "wrote", or
|
|
1412
|
+
"kept" where a file, a folder or a link stands - kept as it is and never followed. InTheWay where a folder on the
|
|
1413
|
+
way is a link or a file (LEADS-ADAPTERS)."""
|
|
1414
|
+
parts = _tree_parts(rel)
|
|
1415
|
+
with _tree_folder(root, parts[:-1], make=True) as folder:
|
|
1416
|
+
try:
|
|
1417
|
+
out = _tree_open(folder, parts[-1], os.O_WRONLY | os.O_CREAT | os.O_EXCL)
|
|
1418
|
+
except FileExistsError:
|
|
1419
|
+
return "kept"
|
|
1420
|
+
with os.fdopen(out, "wb") as fh:
|
|
1421
|
+
fh.write(data)
|
|
1422
|
+
fh.flush()
|
|
1423
|
+
os.fsync(fh.fileno())
|
|
1424
|
+
_tree_sync(folder, os.fsync)
|
|
1425
|
+
return "wrote"
|
|
1426
|
+
|
|
1427
|
+
|
|
1428
|
+
def replace_in_tree(root, rel, data, *, mode=None, _checkpoint=None, _fsync=None):
|
|
1429
|
+
"""Replace the file `rel` in the project at `root` whole, as replace_file_durably does - a temporary under a name no
|
|
1430
|
+
file had, fsynced, renamed over it, the folder fsynced - in the folder held open from the root down: a link at its
|
|
1431
|
+
name is replaced and nothing is written through one. Without `mode` the file keeps the mode it had, or 0644.
|
|
1432
|
+
`_checkpoint` and `_fsync` are replace_file_durably's test seams, at the same steps. InTheWay where a folder on the
|
|
1433
|
+
way is a link or a file, or a folder stands at its name (LEADS-ADAPTERS)."""
|
|
1434
|
+
fsync = _fsync or os.fsync
|
|
1435
|
+
parts = _tree_parts(rel)
|
|
1436
|
+
shown = "/".join(parts)
|
|
1437
|
+
|
|
1438
|
+
def kept_mode(listed):
|
|
1439
|
+
if stat.S_ISDIR(listed.st_mode):
|
|
1440
|
+
raise InTheWay(shown, "is a folder, where a file of the project's goes")
|
|
1441
|
+
return listed.st_mode & 0o7777 if stat.S_ISREG(listed.st_mode) else 0o644
|
|
1442
|
+
|
|
1443
|
+
if not folders_held_open():
|
|
1444
|
+
with _folders_nt(root, parts[:-1], make=True) as folder:
|
|
1445
|
+
path = os.path.join(folder, parts[-1])
|
|
1446
|
+
try:
|
|
1447
|
+
leaf = _open_tree_nt(path, os.O_RDONLY)
|
|
1448
|
+
except FileNotFoundError:
|
|
1449
|
+
kept = 0o644
|
|
1450
|
+
else:
|
|
1451
|
+
try:
|
|
1452
|
+
kept = kept_mode(os.fstat(leaf))
|
|
1453
|
+
finally:
|
|
1454
|
+
os.close(leaf)
|
|
1455
|
+
temporary = os.path.join(folder, f".{parts[-1]}.{os.getpid()}.{os.urandom(4).hex()}.tmp")
|
|
1456
|
+
out = _open_tree_nt(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
|
|
1457
|
+
try:
|
|
1458
|
+
try:
|
|
1459
|
+
_call(_checkpoint, "temporary-opened")
|
|
1460
|
+
_write_all(out, data)
|
|
1461
|
+
_call(_checkpoint, "temporary-written")
|
|
1462
|
+
fsync(out)
|
|
1463
|
+
_call(_checkpoint, "temporary-fsynced")
|
|
1464
|
+
finally:
|
|
1465
|
+
os.close(out)
|
|
1466
|
+
_replace(temporary, path)
|
|
1467
|
+
except BaseException:
|
|
1468
|
+
try:
|
|
1469
|
+
os.unlink(temporary)
|
|
1470
|
+
except OSError:
|
|
1471
|
+
pass
|
|
1472
|
+
raise
|
|
1473
|
+
_call(_checkpoint, "replaced")
|
|
1474
|
+
return
|
|
1475
|
+
fd = _folder_held(root, parts[:-1], make=True)
|
|
1476
|
+
try:
|
|
1477
|
+
name = parts[-1]
|
|
1478
|
+
try:
|
|
1479
|
+
kept = kept_mode(os.stat(name, dir_fd=fd, follow_symlinks=False))
|
|
1480
|
+
except FileNotFoundError:
|
|
1481
|
+
kept = 0o644
|
|
1482
|
+
temporary = f".{name}.{os.getpid()}.{os.urandom(4).hex()}.tmp"
|
|
1483
|
+
out = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
|
1484
|
+
kept if mode is None else mode, dir_fd=fd)
|
|
1485
|
+
try:
|
|
1486
|
+
try:
|
|
1487
|
+
_call(_checkpoint, "temporary-opened")
|
|
1488
|
+
_write_all(out, data)
|
|
1489
|
+
os.fchmod(out, kept if mode is None else mode)
|
|
1490
|
+
_call(_checkpoint, "temporary-written")
|
|
1491
|
+
fsync(out)
|
|
1492
|
+
_call(_checkpoint, "temporary-fsynced")
|
|
1493
|
+
finally:
|
|
1494
|
+
os.close(out)
|
|
1495
|
+
os.replace(temporary, name, src_dir_fd=fd, dst_dir_fd=fd)
|
|
1496
|
+
except BaseException:
|
|
1497
|
+
try:
|
|
1498
|
+
os.unlink(temporary, dir_fd=fd)
|
|
1499
|
+
except OSError:
|
|
1500
|
+
pass
|
|
1501
|
+
raise
|
|
1502
|
+
_call(_checkpoint, "replaced")
|
|
1503
|
+
if _sync_folder_held(fd, fsync):
|
|
1504
|
+
_call(_checkpoint, "directory-fsynced")
|
|
1505
|
+
finally:
|
|
1506
|
+
os.close(fd)
|
|
1507
|
+
|
|
1508
|
+
|
|
1509
|
+
def read_in_tree(root, rel):
|
|
1510
|
+
"""The bytes of the regular file `rel` in the project at `root`, or None where nothing is there. InTheWay where it,
|
|
1511
|
+
or a folder on its way, is a link, or it is no regular file - a FIFO is never waited on (LEADS-ADAPTERS)."""
|
|
1512
|
+
parts = _tree_parts(rel)
|
|
1513
|
+
with _tree_folder(root, parts[:-1], make=False) as folder:
|
|
1514
|
+
if folder is None:
|
|
1515
|
+
return None
|
|
1516
|
+
try:
|
|
1517
|
+
leaf = _tree_open(folder, parts[-1], os.O_RDONLY)
|
|
1518
|
+
except FileNotFoundError:
|
|
1519
|
+
return None
|
|
1520
|
+
except InTheWay as exc:
|
|
1521
|
+
raise InTheWay("/".join(parts), str(exc)[len(str(exc.where)) + 1:]) from exc
|
|
1522
|
+
with os.fdopen(leaf, "rb") as fh:
|
|
1523
|
+
return fh.read()
|
|
1524
|
+
|
|
1525
|
+
|
|
1526
|
+
def append_in_tree(root, rel, data):
|
|
1527
|
+
"""Append only to a single-name regular file opened through no link (LEADS-ADAPTERS)."""
|
|
1528
|
+
parts = _tree_parts(rel)
|
|
1529
|
+
with _tree_folder(root, parts[:-1], make=True) as folder:
|
|
1530
|
+
fd = _tree_open(folder, parts[-1], os.O_WRONLY | os.O_CREAT | os.O_APPEND)
|
|
1531
|
+
try:
|
|
1532
|
+
if os.fstat(fd).st_nlink != 1:
|
|
1533
|
+
raise InTheWay(rel, "is a file with another name")
|
|
1534
|
+
_write_all(fd, data)
|
|
1535
|
+
os.fsync(fd)
|
|
1536
|
+
finally:
|
|
1537
|
+
os.close(fd)
|
|
1538
|
+
_tree_sync(folder, os.fsync)
|
|
1539
|
+
|
|
1540
|
+
|
|
1541
|
+
def _tree_bytes(folder, name):
|
|
1542
|
+
try:
|
|
1543
|
+
fd = _tree_open(folder, name, os.O_RDONLY)
|
|
1544
|
+
except FileNotFoundError:
|
|
1545
|
+
return None
|
|
1546
|
+
with os.fdopen(fd, "rb") as handle:
|
|
1547
|
+
return handle.read()
|
|
1548
|
+
|
|
1549
|
+
|
|
1550
|
+
def _tree_rename(folder, source, target):
|
|
1551
|
+
if isinstance(folder, int):
|
|
1552
|
+
os.replace(source, target, src_dir_fd=folder, dst_dir_fd=folder)
|
|
1553
|
+
else:
|
|
1554
|
+
_replace(os.path.join(folder, source), os.path.join(folder, target))
|
|
1555
|
+
|
|
1556
|
+
|
|
1557
|
+
def _tree_seal(folder, name, identity):
|
|
1558
|
+
"""Read a seal through the held tree, including when its ledger is absent (LEADS-ADAPTERS)."""
|
|
1559
|
+
raw_seal = _tree_bytes(folder, name + ".seal.json")
|
|
1560
|
+
seal = None
|
|
1561
|
+
if raw_seal is not None:
|
|
1562
|
+
try:
|
|
1563
|
+
seal = json.loads(raw_seal.decode(UTF8))
|
|
1564
|
+
except (ValueError, UnicodeError) as exc:
|
|
1565
|
+
raise LedgerCorruption(f"the seal of {identity} cannot be read: {exc}") from exc
|
|
1566
|
+
if not isinstance(seal, dict) or not isinstance(seal.get("retired"), int) or seal["retired"] < 1 \
|
|
1567
|
+
or not isinstance(seal.get("digest"), str):
|
|
1568
|
+
raise LedgerCorruption(f"the seal of {identity} is malformed")
|
|
1569
|
+
return seal
|
|
1570
|
+
|
|
1571
|
+
|
|
1572
|
+
def _tree_replace_bytes(folder, name, data):
|
|
1573
|
+
"""Publish a recovered seal in the folder held for its ledger, even if that folder moved (LEDGER-HISTORY)."""
|
|
1574
|
+
temporary = f".{name}.{os.getpid()}.{os.urandom(4).hex()}.tmp"
|
|
1575
|
+
fd = _tree_open(folder, temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
1576
|
+
try:
|
|
1577
|
+
try:
|
|
1578
|
+
_write_all(fd, data)
|
|
1579
|
+
os.fsync(fd)
|
|
1580
|
+
finally:
|
|
1581
|
+
os.close(fd)
|
|
1582
|
+
_tree_rename(folder, temporary, name)
|
|
1583
|
+
except BaseException:
|
|
1584
|
+
try:
|
|
1585
|
+
_tree_unlink(folder, temporary)
|
|
1586
|
+
except OSError:
|
|
1587
|
+
pass
|
|
1588
|
+
raise
|
|
1589
|
+
_tree_sync(folder, os.fsync)
|
|
1590
|
+
|
|
1591
|
+
|
|
1592
|
+
def _chained_in_tree(root, rel, chain, record=None, every=False):
|
|
1593
|
+
"""The opt-in ledger's read or append, including recovery, under its usual sidecar lock (LEADS-ADAPTERS).
|
|
1594
|
+
|
|
1595
|
+
Keep the JSONL, chain, seals and machine checkpoint unchanged. Only this ledger's filesystem operations
|
|
1596
|
+
change: the lock, ledger and seal are opened in the held folder; truncation and rollback use the same
|
|
1597
|
+
descriptor, never a name an agent can redirect. Other ledger callers keep their existing API.
|
|
1598
|
+
"""
|
|
1599
|
+
nothing = 0 if every else [] # what an empty ledger is to the caller
|
|
1600
|
+
parts = _tree_parts(rel)
|
|
1601
|
+
name = parts[-1]
|
|
1602
|
+
identity = os.path.join(os.path.realpath(root), *parts)
|
|
1603
|
+
# Before the store is read at all: a store inside the tree attests nothing and is never acted on (OPTIN-ATTESTED).
|
|
1604
|
+
if _store_in_tree(root):
|
|
1605
|
+
if record is None:
|
|
1606
|
+
return nothing
|
|
1607
|
+
raise LedgerCorruption(
|
|
1608
|
+
f"the checkpoint store {checkpoint_path()} lies inside the project, where an agent writes: polyctl records "
|
|
1609
|
+
"no opt-in beside it - point POLYCTL_LEDGER_CHECKPOINTS outside the tree, or unset it")
|
|
1610
|
+
recovering = "intent" in _checkpoint_entry(identity, identity=identity)
|
|
1611
|
+
with _tree_folder(root, parts[:-1], make=record is not None or recovering) as folder:
|
|
1612
|
+
if folder is None:
|
|
1613
|
+
_check_missing_chained_ledger(identity, chain, identity=identity)
|
|
1614
|
+
return nothing
|
|
1615
|
+
if record is None and not recovering:
|
|
1616
|
+
# Reading a setting before any opt-in must not create a lock in an otherwise clean project, even
|
|
1617
|
+
# when init refuses during planning. A checkpoint intent needs locked recovery; an in-tree intent
|
|
1618
|
+
# needs locked refusal (LEADS-ADAPTERS, LEDGER-HISTORY).
|
|
1619
|
+
for leaf in (name, name + ".seal.json.pending"):
|
|
1620
|
+
try:
|
|
1621
|
+
_tree_stat(folder, leaf)
|
|
1622
|
+
except FileNotFoundError:
|
|
1623
|
+
continue
|
|
1624
|
+
break
|
|
1625
|
+
else:
|
|
1626
|
+
_check_missing_chained_ledger(identity, chain, seal=_tree_seal(folder, name, identity), identity=identity)
|
|
1627
|
+
return nothing
|
|
1628
|
+
with _exclusive_lock(identity + ".lock", _folder=folder):
|
|
1629
|
+
_settle_seal(identity, chain, CHAIN_PREVIOUS_FIELD, _folder=folder, identity=identity)
|
|
1630
|
+
data = _tree_bytes(folder, name)
|
|
1631
|
+
seal = _load_seal(identity, _folder=folder)
|
|
1632
|
+
if data is None:
|
|
1633
|
+
_check_missing_chained_ledger(identity, chain, seal=seal, identity=identity)
|
|
1634
|
+
data = b""
|
|
1635
|
+
attested = _attested_length(identity, seal, identity=identity)
|
|
1636
|
+
if record is None:
|
|
1637
|
+
records, _ = _jsonl_records(data)
|
|
1638
|
+
if every:
|
|
1639
|
+
return len(records)
|
|
1640
|
+
# Only the rows polyctl recorded are read and checked: a tail past them - well formed or not - is nobody's
|
|
1641
|
+
# record, and never makes polyctl's own unreadable (OPTIN-ATTESTED).
|
|
1642
|
+
rows = [_jsonl_row(body, index, identity) for index, body in records[:attested]]
|
|
1643
|
+
_validate_chained_rows(identity, rows, chain, CHAIN_PREVIOUS_FIELD, seal=seal)
|
|
1644
|
+
_check_committed_length(identity, rows, chain, seal=seal, identity=identity)
|
|
1645
|
+
return rows
|
|
1646
|
+
records, unfinished = _jsonl_records(data)
|
|
1647
|
+
past = len(records) - attested
|
|
1648
|
+
cut_to = None
|
|
1649
|
+
if past > 0:
|
|
1650
|
+
if not _is_interrupted_append(records, attested, identity, chain, identity=identity,
|
|
1651
|
+
unfinished=unfinished):
|
|
1652
|
+
# Appended after them, polyctl's record would vouch for rows it never wrote (OPTIN-ATTESTED). Counted as
|
|
1653
|
+
# a read counts them, before anything parses them.
|
|
1654
|
+
raise LedgerCorruption(f"{rel}: {_unattested_repair(past, attested)}")
|
|
1655
|
+
cut_to = sum(len(line) for line in data.splitlines(keepends=True)[:records[attested][0]])
|
|
1656
|
+
records = records[:attested]
|
|
1657
|
+
rows = [_jsonl_row(body, index, identity) for index, body in records]
|
|
1658
|
+
_validate_chained_rows(identity, rows, chain, CHAIN_PREVIOUS_FIELD, seal=seal)
|
|
1659
|
+
_check_committed_length(identity, rows, chain, seal=seal, identity=identity)
|
|
1660
|
+
retired = seal["retired"] if seal else 0
|
|
1661
|
+
previous = chained_row_digest(rows[-1], chain) if rows else (seal["digest"] if seal else None)
|
|
1662
|
+
chained = {CHAIN_PREVIOUS_FIELD: previous, "ordinal": retired + len(rows) + 1}
|
|
1663
|
+
chained.update(record)
|
|
1664
|
+
payload = _encode_jsonl(chained)
|
|
1665
|
+
digest = chained_row_digest(chained, chain)
|
|
1666
|
+
created = False
|
|
1667
|
+
try:
|
|
1668
|
+
fd = _tree_open(folder, name, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o644)
|
|
1669
|
+
created = True
|
|
1670
|
+
except FileExistsError:
|
|
1671
|
+
fd = _tree_open(folder, name, os.O_RDWR)
|
|
1672
|
+
remove_created = created
|
|
1673
|
+
try:
|
|
1674
|
+
if os.fstat(fd).st_nlink != 1:
|
|
1675
|
+
raise InTheWay(rel, "has another name; polyctl appends only to the project's own file")
|
|
1676
|
+
# Refuse a different leaf swapped in since the read; a hard link is refused before any truncation.
|
|
1677
|
+
with os.fdopen(os.dup(fd), "rb") as handle:
|
|
1678
|
+
current = handle.read()
|
|
1679
|
+
if current != data:
|
|
1680
|
+
raise InTheWay(rel, "changed while polyctl read it; retry the opt-in")
|
|
1681
|
+
if cut_to is not None:
|
|
1682
|
+
boundary = cut_to
|
|
1683
|
+
else:
|
|
1684
|
+
boundary = len(data) if not data or data.endswith((b"\n", b"\r")) else \
|
|
1685
|
+
max(data.rfind(b"\n"), data.rfind(b"\r")) + 1
|
|
1686
|
+
os.ftruncate(fd, boundary)
|
|
1687
|
+
if boundary != len(data):
|
|
1688
|
+
os.fsync(fd)
|
|
1689
|
+
if cut_to is not None:
|
|
1690
|
+
_clear_append_intent(identity, identity=identity)
|
|
1691
|
+
os.lseek(fd, boundary, os.SEEK_SET)
|
|
1692
|
+
_record_append_intent(identity, digest, identity=identity)
|
|
1693
|
+
try:
|
|
1694
|
+
_write_all(fd, payload)
|
|
1695
|
+
os.fsync(fd)
|
|
1696
|
+
if created:
|
|
1697
|
+
_tree_sync(folder, os.fsync)
|
|
1698
|
+
_record_committed_length(identity, retired + len(rows) + 1, digest, identity=identity)
|
|
1699
|
+
except BaseException:
|
|
1700
|
+
os.ftruncate(fd, boundary)
|
|
1701
|
+
os.fsync(fd)
|
|
1702
|
+
raise
|
|
1703
|
+
remove_created = False
|
|
1704
|
+
finally:
|
|
1705
|
+
held = os.fstat(fd)
|
|
1706
|
+
os.close(fd)
|
|
1707
|
+
if remove_created:
|
|
1708
|
+
try:
|
|
1709
|
+
named = _tree_stat(folder, name)
|
|
1710
|
+
if (held.st_dev, held.st_ino) == (named.st_dev, named.st_ino):
|
|
1711
|
+
_tree_unlink(folder, name)
|
|
1712
|
+
except OSError:
|
|
1713
|
+
pass
|
|
1714
|
+
return chained
|
|
1715
|
+
|
|
1716
|
+
|
|
1717
|
+
def append_chained_in_tree(root, rel, record, chain):
|
|
1718
|
+
"""Append an opt-in without following a folder or leaf link (LEADS-ADAPTERS)."""
|
|
1719
|
+
return _chained_in_tree(root, rel, chain, record)
|
|
1720
|
+
|
|
1721
|
+
|
|
1722
|
+
def read_chained_in_tree(root, rel, chain):
|
|
1723
|
+
"""Read opt-ins under the same held-folder lock as their append (LEADS-ADAPTERS): the rows this machine's checkpoint
|
|
1724
|
+
store records polyctl appending, and no others (OPTIN-ATTESTED).
|
|
1725
|
+
|
|
1726
|
+
The chain proves no committed row was changed and the recorded length that none was cut off; neither says who
|
|
1727
|
+
wrote a row past that length. A person's opt-in weakens a guarantee, and the implementer can write any file in
|
|
1728
|
+
the tree: the first row of a chain needs no digest at all, and one past polyctl's last append needs only the digest of
|
|
1729
|
+
the row before it. A row there - in a ledger this machine never recorded, or added after polyctl's last append - is
|
|
1730
|
+
nobody's record.
|
|
1731
|
+
"""
|
|
1732
|
+
return _chained_in_tree(root, rel, chain)
|
|
1733
|
+
|
|
1734
|
+
|
|
1735
|
+
def unattested_in_tree(root, rel, chain):
|
|
1736
|
+
"""(rows this machine's checkpoint store does not record polyctl appending, rows it does) of an in-tree ledger
|
|
1737
|
+
(OPTIN-ATTESTED)."""
|
|
1738
|
+
attested = len(_chained_in_tree(root, rel, chain))
|
|
1739
|
+
return max(0, _chained_in_tree(root, rel, chain, every=True) - attested), attested
|
|
1740
|
+
|
|
1741
|
+
|
|
1742
|
+
def _store_in_tree(root):
|
|
1743
|
+
"""Whether this machine's checkpoint store lies inside the project's tree (OPTIN-ATTESTED).
|
|
1744
|
+
|
|
1745
|
+
POLYCTL_LEDGER_CHECKPOINTS moves the store, and the implementer can name it on any command it runs: a store inside the
|
|
1746
|
+
tree is a file it writes, so a row it wrote and a count it wrote beside it would vouch for each other. Such a store
|
|
1747
|
+
attests nothing, and polyctl appends no opt-in beside it, as `polyctl doctor` already names the variable (PYTHON-ENV).
|
|
1748
|
+
|
|
1749
|
+
Compared by what the folders are, not how their names are spelled: a case-insensitive file system answers the
|
|
1750
|
+
same folder to `/A/Proj` and `/a/proj`, and the real path keeps whichever spelling it was given. Each folder on
|
|
1751
|
+
the store's path, as named and as resolved, is held to the root's device and inode, so neither case nor a link
|
|
1752
|
+
moves it out.
|
|
1753
|
+
"""
|
|
1754
|
+
try:
|
|
1755
|
+
top = os.stat(root)
|
|
1756
|
+
except OSError:
|
|
1757
|
+
return False
|
|
1758
|
+
named = checkpoint_path()
|
|
1759
|
+
if os.pardir in re.split(r"[\\/]+", named):
|
|
1760
|
+
# `a/link/..` is the link target's parent to the file system and `a` to a path read as text: either reading
|
|
1761
|
+
# can hide the other, so a store named with `..` is taken as the tree's.
|
|
1762
|
+
return True
|
|
1763
|
+
given = named if os.path.isabs(named) else os.path.join(os.getcwd(), named)
|
|
1764
|
+
# The path as named and as resolved, both: one through a link the implementer keeps in the tree is the tree's,
|
|
1765
|
+
# wherever the link points as polyctl looks, since the link can point elsewhere by the time the store is read.
|
|
1766
|
+
for path in (given, os.path.realpath(named)):
|
|
1767
|
+
if not top.st_ino:
|
|
1768
|
+
# A file system that numbers no file: the paths, case folded where the platform folds case.
|
|
1769
|
+
store, folder = os.path.normcase(path), os.path.normcase(os.path.realpath(root))
|
|
1770
|
+
if store == folder or store.startswith(folder.rstrip(os.sep) + os.sep):
|
|
1771
|
+
return True
|
|
1772
|
+
continue
|
|
1773
|
+
here = os.path.dirname(path)
|
|
1774
|
+
while True:
|
|
1775
|
+
try:
|
|
1776
|
+
held = os.stat(here)
|
|
1777
|
+
except OSError:
|
|
1778
|
+
held = None
|
|
1779
|
+
if held is not None and (held.st_dev, held.st_ino) == (top.st_dev, top.st_ino):
|
|
1780
|
+
return True
|
|
1781
|
+
parent = os.path.dirname(here)
|
|
1782
|
+
if parent == here:
|
|
1783
|
+
break
|
|
1784
|
+
here = parent
|
|
1785
|
+
return False
|
|
1786
|
+
|
|
1787
|
+
|
|
1788
|
+
def _ledger_store_in_tree(path):
|
|
1789
|
+
"""Whether `path` is a project ledger under `.polyctl/ledger/` whose checkpoint store lies inside that project."""
|
|
1790
|
+
ledger_dir = os.path.dirname(os.path.abspath(path))
|
|
1791
|
+
polyctl_dir = os.path.dirname(ledger_dir)
|
|
1792
|
+
if os.path.basename(ledger_dir) == "ledger" and os.path.basename(polyctl_dir) == ".polyctl":
|
|
1793
|
+
return _store_in_tree(os.path.dirname(polyctl_dir))
|
|
1794
|
+
return False
|
|
1795
|
+
|
|
1796
|
+
|
|
1797
|
+
def _unattested_repair(foreign, attested):
|
|
1798
|
+
"""What a person does about rows past polyctl's record, keeping the ones polyctl did record (OPTIN-ATTESTED)."""
|
|
1799
|
+
keep = (f"the first {attested} row(s) are polyctl's own and stay; a person removes exactly the last {foreign} line(s) "
|
|
1800
|
+
"of the file") if attested else ("no row of it is polyctl's own; a person moves the file aside, or attests a "
|
|
1801
|
+
"restored backup with polyctl restore <source> --by <name> --ask")
|
|
1802
|
+
return (f"{foreign} row(s) the checkpoint store of this machine ({checkpoint_path()}) does not record polyctl "
|
|
1803
|
+
f"appending - something other than polyctl wrote them, or an append did not finish. They turn nothing off and "
|
|
1804
|
+
f"polyctl appends nothing after them: {keep}, and records again what is missing")
|
|
1805
|
+
|
|
1806
|
+
|
|
1807
|
+
def _attested_length(path, seal, *, identity=None):
|
|
1808
|
+
"""How many of a ledger's live rows this machine's checkpoint store records polyctl committing (OPTIN-ATTESTED)."""
|
|
1809
|
+
count = _checkpoint_entry(path, identity=identity).get("count")
|
|
1810
|
+
return max(0, (count if isinstance(count, int) else 0) - (seal["retired"] if seal else 0))
|
|
1811
|
+
|
|
1812
|
+
|
|
1813
|
+
def _is_interrupted_append(records, attested, path, chain, *, identity=None, unfinished=None):
|
|
1814
|
+
"""Whether the single record past `attested` is the one this machine recorded an intent to append (LEDGER-ATTESTED)."""
|
|
1815
|
+
if unfinished is not None or len(records) - attested != 1:
|
|
1816
|
+
return False
|
|
1817
|
+
intent = _checkpoint_entry(path, identity=identity).get("append")
|
|
1818
|
+
if not isinstance(intent, str) or not intent:
|
|
1819
|
+
return False
|
|
1820
|
+
index, body = records[attested]
|
|
1821
|
+
try:
|
|
1822
|
+
row = _jsonl_row(body, index, path)
|
|
1823
|
+
return isinstance(row, dict) and chained_row_digest(row, chain) == intent
|
|
1824
|
+
except (LedgerCorruption, TypeError, ValueError):
|
|
1825
|
+
return False
|
|
1826
|
+
|
|
1827
|
+
|
|
1828
|
+
def _repair_partial_tail(path, fsync, checkpoint, *, limit=None):
|
|
1829
|
+
"""Cut off the uncommitted final record an interrupted or failed append left.
|
|
1830
|
+
|
|
1831
|
+
It is never completed: its writer did not return, so it was never committed (#68).
|
|
1832
|
+
"""
|
|
1833
|
+
if not os.path.exists(path) or os.path.getsize(path) == 0:
|
|
1834
|
+
return "clean"
|
|
1835
|
+
|
|
1836
|
+
# Validate every complete row before touching the tail. Without this pass an
|
|
1837
|
+
# invalid earlier row followed by a partial tail could be retained while a
|
|
1838
|
+
# new record was appended, falsely reporting a successful durable write on
|
|
1839
|
+
# an already-corrupt ledger.
|
|
1840
|
+
data = open(path, "rb").read()
|
|
1841
|
+
if limit is None:
|
|
1842
|
+
_read_jsonl_unlocked(path, allow_partial_tail=True)
|
|
1843
|
+
else:
|
|
1844
|
+
for index, body in _jsonl_records(data)[0][:limit]:
|
|
1845
|
+
_jsonl_row(body, index, path)
|
|
1846
|
+
if data.endswith((b"\n", b"\r")):
|
|
1847
|
+
return "clean"
|
|
1848
|
+
|
|
1849
|
+
boundary = max(data.rfind(b"\n"), data.rfind(b"\r")) + 1
|
|
1850
|
+
with open(path, "r+b", buffering=0) as handle:
|
|
1851
|
+
handle.truncate(boundary)
|
|
1852
|
+
fsync(handle.fileno())
|
|
1853
|
+
_call(checkpoint, "truncated-tail")
|
|
1854
|
+
return "truncated-tail"
|
|
1855
|
+
|
|
1856
|
+
|
|
1857
|
+
def _encode_jsonl(record):
|
|
1858
|
+
# The same inputs the chain digest accepts - ASCII escapes, no NaN - so no row
|
|
1859
|
+
# is written that could never be digested, and a path Git hands over
|
|
1860
|
+
# undecodable is escaped rather than refused (#68).
|
|
1861
|
+
return (json.dumps(
|
|
1862
|
+
record, ensure_ascii=True, allow_nan=False, separators=(",", ":")
|
|
1863
|
+
) + "\n").encode("ascii")
|
|
1864
|
+
|
|
1865
|
+
|
|
1866
|
+
def _append_payload_unlocked(path, parent, payload, write, fsync, checkpoint):
|
|
1867
|
+
"""Append one encoded row while the caller owns the sidecar lock.
|
|
1868
|
+
|
|
1869
|
+
All or nothing (#68): a failure at any step - a short write, the file fsync, the
|
|
1870
|
+
directory fsync - cuts the written bytes off again, and removes a file this
|
|
1871
|
+
append created, before the error goes to the caller. Returns the same undo for
|
|
1872
|
+
a caller whose own next step fails while it still holds the lock.
|
|
1873
|
+
"""
|
|
1874
|
+
created = not os.path.exists(path)
|
|
1875
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, "O_BINARY", 0)
|
|
1876
|
+
fd = os.open(path, flags, 0o644)
|
|
1877
|
+
size = os.fstat(fd).st_size
|
|
1878
|
+
|
|
1879
|
+
def undo():
|
|
1880
|
+
try:
|
|
1881
|
+
if created:
|
|
1882
|
+
os.remove(path)
|
|
1883
|
+
return
|
|
1884
|
+
os.truncate(path, size)
|
|
1885
|
+
handle = os.open(path, os.O_WRONLY | getattr(os, "O_BINARY", 0))
|
|
1886
|
+
try:
|
|
1887
|
+
os.fsync(handle)
|
|
1888
|
+
finally:
|
|
1889
|
+
os.close(handle)
|
|
1890
|
+
except OSError:
|
|
1891
|
+
pass
|
|
1892
|
+
|
|
1893
|
+
try:
|
|
1894
|
+
_call(checkpoint, "opened")
|
|
1895
|
+
remaining = memoryview(payload)
|
|
1896
|
+
while remaining:
|
|
1897
|
+
try:
|
|
1898
|
+
count = write(fd, remaining)
|
|
1899
|
+
except InterruptedError:
|
|
1900
|
+
continue
|
|
1901
|
+
if not isinstance(count, int) or count <= 0 or count > len(remaining):
|
|
1902
|
+
raise OSError(errno.EIO, "append made no forward progress")
|
|
1903
|
+
remaining = remaining[count:]
|
|
1904
|
+
_call(checkpoint, "written")
|
|
1905
|
+
fsync(fd)
|
|
1906
|
+
_call(checkpoint, "fsynced")
|
|
1907
|
+
except BaseException:
|
|
1908
|
+
os.close(fd)
|
|
1909
|
+
undo()
|
|
1910
|
+
raise
|
|
1911
|
+
os.close(fd)
|
|
1912
|
+
|
|
1913
|
+
if created:
|
|
1914
|
+
try:
|
|
1915
|
+
synced = _sync_directory(parent, fsync)
|
|
1916
|
+
except BaseException:
|
|
1917
|
+
undo()
|
|
1918
|
+
raise
|
|
1919
|
+
if synced:
|
|
1920
|
+
_call(checkpoint, "directory-fsynced")
|
|
1921
|
+
return undo
|
|
1922
|
+
|
|
1923
|
+
|
|
1924
|
+
def append_jsonl(path, record, timeout=10.0, *, _checkpoint=None,
|
|
1925
|
+
_write=None, _fsync=None):
|
|
1926
|
+
"""Append one JSON object and durably persist it before returning.
|
|
1927
|
+
|
|
1928
|
+
``_checkpoint``, ``_write`` and ``_fsync`` are deliberately private test
|
|
1929
|
+
seams. They let the suite stop a real subprocess between storage barriers
|
|
1930
|
+
and inject short writes or I/O failures without a production crash switch.
|
|
1931
|
+
"""
|
|
1932
|
+
parent = os.path.dirname(path) or "."
|
|
1933
|
+
os.makedirs(parent, exist_ok=True)
|
|
1934
|
+
payload = _encode_jsonl(record)
|
|
1935
|
+
write = _write or os.write
|
|
1936
|
+
fsync = _fsync or os.fsync
|
|
1937
|
+
|
|
1938
|
+
with _exclusive_lock(path + ".lock", timeout=timeout):
|
|
1939
|
+
_call(_checkpoint, "locked")
|
|
1940
|
+
_repair_partial_tail(path, fsync, _checkpoint)
|
|
1941
|
+
_append_payload_unlocked(
|
|
1942
|
+
path, parent, payload, write, fsync, _checkpoint
|
|
1943
|
+
)
|
|
1944
|
+
return record
|
|
1945
|
+
|
|
1946
|
+
|
|
1947
|
+
def append_chained_jsonl(path, record, chain, timeout=10.0, *,
|
|
1948
|
+
previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False, prepare=None,
|
|
1949
|
+
_checkpoint=None, _write=None, _fsync=None):
|
|
1950
|
+
"""Atomically select a predecessor, append, and persist one chained row.
|
|
1951
|
+
|
|
1952
|
+
Existing committed rows are validated before a successor is constructed.
|
|
1953
|
+
Predecessor selection and append share one lock, so concurrent writers form
|
|
1954
|
+
one linear chain instead of siblings that name the same predecessor.
|
|
1955
|
+
`prepare`, given the committed rows, returns fields to set on the record under
|
|
1956
|
+
that lock: what must be unique across writers is decided where they take turns.
|
|
1957
|
+
"""
|
|
1958
|
+
if not isinstance(record, dict):
|
|
1959
|
+
raise TypeError("a chained ledger row must be a JSON object")
|
|
1960
|
+
if not isinstance(previous_field, str) or not previous_field:
|
|
1961
|
+
raise ValueError("previous_field must be a non-empty string")
|
|
1962
|
+
if previous_field in record or "ordinal" in record:
|
|
1963
|
+
raise ValueError(f"caller must not set the {previous_field!r} or 'ordinal' chain fields")
|
|
1964
|
+
gate = _is_gate_ledger(path, chain)
|
|
1965
|
+
if gate and _ledger_store_in_tree(path):
|
|
1966
|
+
raise LedgerCorruption(
|
|
1967
|
+
f"the checkpoint store {checkpoint_path()} lies inside the project, where an agent writes: polyctl records "
|
|
1968
|
+
"no row beside it - point POLYCTL_LEDGER_CHECKPOINTS outside the tree, or unset it")
|
|
1969
|
+
|
|
1970
|
+
parent = os.path.dirname(path) or "."
|
|
1971
|
+
os.makedirs(parent, exist_ok=True)
|
|
1972
|
+
write = _write or os.write
|
|
1973
|
+
fsync = _fsync or os.fsync
|
|
1974
|
+
|
|
1975
|
+
with _exclusive_lock(path + ".lock", timeout=timeout):
|
|
1976
|
+
_call(_checkpoint, "locked")
|
|
1977
|
+
_settle_seal(path, chain, previous_field, legacy_prefix)
|
|
1978
|
+
seal = _load_seal(path)
|
|
1979
|
+
if not os.path.exists(path):
|
|
1980
|
+
_check_missing_chained_ledger(path, chain, seal=seal)
|
|
1981
|
+
if gate:
|
|
1982
|
+
attested = _attested_length(path, seal)
|
|
1983
|
+
if os.path.exists(path):
|
|
1984
|
+
with open(path, "rb") as handle:
|
|
1985
|
+
data = handle.read()
|
|
1986
|
+
else:
|
|
1987
|
+
data = b""
|
|
1988
|
+
records, unfinished = _jsonl_records(data)
|
|
1989
|
+
past = len(records) - attested
|
|
1990
|
+
if past > 0:
|
|
1991
|
+
if not _is_interrupted_append(records, attested, path, chain, unfinished=unfinished):
|
|
1992
|
+
# Appended after them, polyctl's record would vouch for rows it never wrote (OPTIN-ATTESTED,
|
|
1993
|
+
# LEDGER-ATTESTED). Counted as a read counts them, before anything parses or truncates them.
|
|
1994
|
+
raise LedgerCorruption(f"{path}: {_unattested_repair(past, attested)}")
|
|
1995
|
+
rows = [_jsonl_row(body, index, path) for index, body in records[:attested]]
|
|
1996
|
+
_validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
|
|
1997
|
+
_check_committed_length(path, rows, chain, seal=seal)
|
|
1998
|
+
boundary = sum(len(line) for line in data.splitlines(keepends=True)[:records[attested][0]])
|
|
1999
|
+
with open(path, "r+b", buffering=0) as handle:
|
|
2000
|
+
handle.truncate(boundary)
|
|
2001
|
+
fsync(handle.fileno())
|
|
2002
|
+
_call(_checkpoint, "truncated-tail")
|
|
2003
|
+
_clear_append_intent(path)
|
|
2004
|
+
with open(path, "rb") as handle:
|
|
2005
|
+
data = handle.read()
|
|
2006
|
+
else:
|
|
2007
|
+
rows = [_jsonl_row(body, index, path) for index, body in records[:attested]]
|
|
2008
|
+
_validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
|
|
2009
|
+
_check_committed_length(path, rows, chain, seal=seal)
|
|
2010
|
+
_repair_partial_tail(path, fsync, _checkpoint, limit=attested)
|
|
2011
|
+
if os.path.exists(path):
|
|
2012
|
+
with open(path, "rb") as handle:
|
|
2013
|
+
data = handle.read()
|
|
2014
|
+
else:
|
|
2015
|
+
data = b""
|
|
2016
|
+
else:
|
|
2017
|
+
_repair_partial_tail(path, fsync, _checkpoint)
|
|
2018
|
+
if os.path.exists(path):
|
|
2019
|
+
with open(path, "rb") as handle:
|
|
2020
|
+
data = handle.read()
|
|
2021
|
+
else:
|
|
2022
|
+
data = b""
|
|
2023
|
+
rows = _jsonl_rows(data, path, allow_partial_tail=False)
|
|
2024
|
+
_validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
|
|
2025
|
+
_check_committed_length(path, rows, chain, seal=seal)
|
|
2026
|
+
retired = seal["retired"] if seal else 0
|
|
2027
|
+
previous = chained_row_digest(rows[-1], chain) if rows else (seal["digest"] if seal else None)
|
|
2028
|
+
chained = {previous_field: previous, "ordinal": retired + len(rows) + 1}
|
|
2029
|
+
chained.update(record)
|
|
2030
|
+
if prepare is not None:
|
|
2031
|
+
fields = prepare(rows) or {}
|
|
2032
|
+
if previous_field in fields or "ordinal" in fields:
|
|
2033
|
+
raise ValueError(f"prepare must not set the {previous_field!r} or 'ordinal' chain fields")
|
|
2034
|
+
chained.update(fields)
|
|
2035
|
+
payload = _encode_jsonl(chained)
|
|
2036
|
+
digest = chained_row_digest(chained, chain)
|
|
2037
|
+
_record_append_intent(path, digest)
|
|
2038
|
+
try:
|
|
2039
|
+
undo = _append_payload_unlocked(
|
|
2040
|
+
path, parent, payload, write, fsync, _checkpoint
|
|
2041
|
+
)
|
|
2042
|
+
except BaseException:
|
|
2043
|
+
try:
|
|
2044
|
+
_clear_append_intent(path)
|
|
2045
|
+
except OSError:
|
|
2046
|
+
pass
|
|
2047
|
+
raise
|
|
2048
|
+
try:
|
|
2049
|
+
_record_committed_length(path, retired + len(rows) + 1, digest)
|
|
2050
|
+
except BaseException:
|
|
2051
|
+
# A row whose length could not be recorded is taken back, so the
|
|
2052
|
+
# caller's failure and the file agree (#68).
|
|
2053
|
+
undo()
|
|
2054
|
+
raise
|
|
2055
|
+
return chained
|